Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Lots of fixes to poor daggyr implementation, added debugging messages

Browse Source
This commit is contained in:
Ian Roddis
2021-12-21 16:49:45 -04:00
parent 505ffb02bd
commit d90f49b2bb
6 changed files with 142 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
daggyd/daggyd/daggyd.cpp
View File

@@ -119,101 +119,99 @@ namespace dl = daggy::loggers::dag_run;
std::unique_ptr<dl::DAGRunLogger> loggerFactory(const rj::Value &config)
{
if (config.HasMember("logger")) {
const auto &logConf = config["logger"];
if (!logConf.IsObject())
throw std::runtime_error("logger config is not an object");
if (!logConf.HasMember("name"))
throw std::runtime_error("logger config is missing logger name");
if (!logConf.HasMember("config"))
throw std::runtime_error("logger config is missing logger config");
if (!config.HasMember("logger"))
return std::make_unique<dl::OStreamLogger>(std::cout);
std::string name = logConf["name"].GetString();
const auto &logConfig = logConf["config"];
if (name == "OStreamLogger") {
if (logConfig.HasMember("file")) {
std::string fn = logConfig["file"].GetString();
if (fn == "-")
return std::make_unique<dl::OStreamLogger>(std::cout);
const auto &logConf = config["logger"];
if (!logConf.IsObject())
throw std::runtime_error("logger config is not an object");
if (!logConf.HasMember("name"))
throw std::runtime_error("logger config is missing logger name");
if (!logConf.HasMember("config"))
throw std::runtime_error("logger config is missing logger config");
std::ofstream ofh(logConfig["file"].GetString());
return std::make_unique<dl::OStreamLogger>(ofh);
}
std::string name = logConf["name"].GetString();
const auto &logConfig = logConf["config"];
if (name == "OStreamLogger") {
if (logConfig.HasMember("file")) {
std::string fn = logConfig["file"].GetString();
if (fn == "-")
return std::make_unique<dl::OStreamLogger>(std::cout);
std::ofstream ofh(logConfig["file"].GetString());
return std::make_unique<dl::OStreamLogger>(ofh);
}
#ifdef DAGGY_ENABLE_REDIS
else if (name == "RedisLogger") {
std::string host = "localhost";
uint16_t port = 6379;
std::string prefix = "daggy";
if (logConfig.HasMember("prefix"))
prefix = logConfig["prefix"].GetString();
if (logConfig.HasMember("host"))
host = logConfig["host"].GetString();
if (logConfig.HasMember("port"))
port = logConfig["port"].GetInt();
return std::make_unique<dl::RedisLogger>(prefix, host, port);
}
#endif
else
throw std::runtime_error("Unknown logger type: " + name);
}
return std::make_unique<dl::OStreamLogger>(std::cout);
#ifdef DAGGY_ENABLE_REDIS
else if (name == "RedisLogger") {
std::string host = "localhost";
uint16_t port = 6379;
std::string prefix = "daggy";
if (logConfig.HasMember("prefix"))
prefix = logConfig["prefix"].GetString();
if (logConfig.HasMember("host"))
host = logConfig["host"].GetString();
if (logConfig.HasMember("port"))
port = logConfig["port"].GetInt();
return std::make_unique<dl::RedisLogger>(prefix, host, port);
}
#endif
throw std::runtime_error("Unknown logger type: " + name);
}
namespace de = daggy::executors::task;
std::unique_ptr<de::TaskExecutor> executorFactory(const rj::Value &config)
{
if (config.HasMember("executor")) {
const auto &execConf = config["executor"];
if (!execConf.IsObject())
throw std::runtime_error("Executor config is not an object");
if (!execConf.HasMember("name"))
throw std::runtime_error("Executor config is missing name");
if (!execConf.HasMember("config"))
throw std::runtime_error("Executor config is missing config");
std::string name = execConf["name"].GetString();
const auto &execConfig = execConf["config"];
if (!config.HasMember("executor"))
return std::make_unique<de::ForkingTaskExecutor>(10);
if (name == "ForkingTaskExecutor") {
size_t threads = 10;
if (execConfig.HasMember("threads"))
threads = execConfig["threads"].GetInt64();
return std::make_unique<de::ForkingTaskExecutor>(threads);
}
const auto &execConf = config["executor"];
if (!execConf.IsObject())
throw std::runtime_error("Executor config is not an object");
if (!execConf.HasMember("name"))
throw std::runtime_error("Executor config is missing name");
if (!execConf.HasMember("config"))
throw std::runtime_error("Executor config is missing config");
std::string name = execConf["name"].GetString();
const auto &execConfig = execConf["config"];
if (name == "ForkingTaskExecutor") {
size_t threads = 10;
if (execConfig.HasMember("threads"))
threads = execConfig["threads"].GetInt64();
return std::make_unique<de::ForkingTaskExecutor>(threads);
}
#ifdef DAGGY_ENABLE_SLURM
else if (name == "SlurmTaskExecutor") {
return std::make_unique<de::SlurmTaskExecutor>();
}
else if (name == "SlurmTaskExecutor") {
return std::make_unique<de::SlurmTaskExecutor>();
}
#endif
else if (name == "DaggyRunnerTaskExecutor") {
if (!execConfig.HasMember("runners"))
throw std::runtime_error(
"DaggyRunnerExecutor config needs at least one remote runner");
else if (name == "DaggyRunnerTaskExecutor") {
if (!execConfig.HasMember("runners"))
throw std::runtime_error(
"DaggyRunnerExecutor config needs at least one remote runner");
auto exe = std::make_unique<de::DaggyRunnerTaskExecutor>();
auto exe = std::make_unique<de::DaggyRunnerTaskExecutor>();
const auto &runners = execConfig["runners"];
if (!runners.IsArray()) {
const auto &runners = execConfig["runners"];
if (!runners.IsArray())
throw std::runtime_error(
"DaggyRunnerExecutor runners must be an array of urls");
for (size_t i = 0; i < runners.Size(); ++i) {
if (!runners[i].IsString())
throw std::runtime_error(
"DaggyRunnerExecutor runners must be an array of urls");
for (size_t i = 0; i < runners.Size(); ++i) {
if (!runners[i].IsString())
throw std::runtime_error(
"DaggyRunnerExecutor runners must be an array of urls");
exe->addRunner(runners[i].GetString());
}
return exe;
}
exe->addRunner(runners[i].GetString());
}
else
throw std::runtime_error("Unknown executor type: " + name);
return exe;
}
return std::make_unique<de::ForkingTaskExecutor>(10);
throw std::runtime_error("Unknown executor type: " + name);
}
int main(int argc, char **argv)