- Running project through clang-tidy formatter.
This commit is contained in:
@@ -15,9 +15,11 @@ namespace rj = rapidjson;
|
|||||||
namespace daggy {
|
namespace daggy {
|
||||||
// Parameters
|
// Parameters
|
||||||
ParameterValues parametersFromJSON(const std::string &jsonSpec);
|
ParameterValues parametersFromJSON(const std::string &jsonSpec);
|
||||||
|
|
||||||
ParameterValues parametersFromJSON(const rj::Document &spec);
|
ParameterValues parametersFromJSON(const rj::Document &spec);
|
||||||
|
|
||||||
// Tasks
|
// Tasks
|
||||||
std::vector<Task> tasksFromJSON(const std::string &jsonSpec, const ParameterValues ¶meters = {});
|
std::vector<Task> tasksFromJSON(const std::string &jsonSpec, const ParameterValues ¶meters = {});
|
||||||
|
|
||||||
std::vector<Task> tasksFromJSON(const rj::Document &spec, const ParameterValues ¶meters = {});
|
std::vector<Task> tasksFromJSON(const rj::Document &spec, const ParameterValues ¶meters = {});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,7 @@ namespace daggy {
|
|||||||
class ForkingTaskExecutor : public TaskExecutor {
|
class ForkingTaskExecutor : public TaskExecutor {
|
||||||
public:
|
public:
|
||||||
ForkingTaskExecutor(size_t nThreads)
|
ForkingTaskExecutor(size_t nThreads)
|
||||||
: TaskExecutor(nThreads)
|
: TaskExecutor(nThreads) {}
|
||||||
{}
|
|
||||||
|
|
||||||
const std::string getName() const override { return "ForkingTaskExecutor"; }
|
const std::string getName() const override { return "ForkingTaskExecutor"; }
|
||||||
|
|
||||||
|
|||||||
@@ -56,12 +56,16 @@ namespace daggy {
|
|||||||
public:
|
public:
|
||||||
// Execution
|
// Execution
|
||||||
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) = 0;
|
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) = 0;
|
||||||
|
|
||||||
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) = 0;
|
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) = 0;
|
||||||
|
|
||||||
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) = 0;
|
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) = 0;
|
||||||
|
|
||||||
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) = 0;
|
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) = 0;
|
||||||
|
|
||||||
// Querying
|
// Querying
|
||||||
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) = 0;
|
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) = 0;
|
||||||
|
|
||||||
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId) = 0;
|
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId) = 0;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,12 +40,16 @@ namespace daggy {
|
|||||||
|
|
||||||
// Execution
|
// Execution
|
||||||
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
|
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
|
||||||
|
|
||||||
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
|
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
|
||||||
|
|
||||||
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
|
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
|
||||||
|
|
||||||
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
|
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
|
||||||
|
|
||||||
// Querying
|
// Querying
|
||||||
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
|
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
|
||||||
|
|
||||||
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
|
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@@ -56,7 +60,9 @@ namespace daggy {
|
|||||||
// std::unordered_map<fs::path, std::mutex> runLocks;
|
// std::unordered_map<fs::path, std::mutex> runLocks;
|
||||||
|
|
||||||
inline const fs::path getCurrentPath() const;
|
inline const fs::path getCurrentPath() const;
|
||||||
|
|
||||||
inline const fs::path getRunsRoot() const;
|
inline const fs::path getRunsRoot() const;
|
||||||
|
|
||||||
inline const fs::path getRunRoot(DAGRunID runID) const;
|
inline const fs::path getRunRoot(DAGRunID runID) const;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,12 +18,16 @@ namespace daggy {
|
|||||||
|
|
||||||
// Execution
|
// Execution
|
||||||
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
|
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
|
||||||
|
|
||||||
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
|
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
|
||||||
|
|
||||||
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
|
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
|
||||||
|
|
||||||
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
|
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
|
||||||
|
|
||||||
// Querying
|
// Querying
|
||||||
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
|
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
|
||||||
|
|
||||||
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
|
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -24,7 +24,9 @@ namespace daggy {
|
|||||||
std::vector<std::string> values;
|
std::vector<std::string> values;
|
||||||
for (size_t i = 0; i < it->value.Size(); ++i) {
|
for (size_t i = 0; i < it->value.Size(); ++i) {
|
||||||
if (!it->value[i].IsString()) {
|
if (!it->value[i].IsString()) {
|
||||||
throw std::runtime_error("Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) + " is not a string.");
|
throw std::runtime_error(
|
||||||
|
"Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) +
|
||||||
|
" is not a string.");
|
||||||
}
|
}
|
||||||
values.emplace_back(it->value[i].GetString());
|
values.emplace_back(it->value[i].GetString());
|
||||||
}
|
}
|
||||||
@@ -32,7 +34,8 @@ namespace daggy {
|
|||||||
} else if (it->value.IsString()) {
|
} else if (it->value.IsString()) {
|
||||||
parameters[name] = it->value.GetString();
|
parameters[name] = it->value.GetString();
|
||||||
} else {
|
} else {
|
||||||
throw std::runtime_error("Attribute for " + std::string{it->name.GetString()} + " is not a string or an array.");
|
throw std::runtime_error(
|
||||||
|
"Attribute for " + std::string{it->name.GetString()} + " is not a string or an array.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return parameters;
|
return parameters;
|
||||||
@@ -77,7 +80,8 @@ namespace daggy {
|
|||||||
uint8_t maxRetries = 0;
|
uint8_t maxRetries = 0;
|
||||||
if (taskSpec.HasMember("maxRetries")) { maxRetries = taskSpec["maxRetries"].GetInt(); }
|
if (taskSpec.HasMember("maxRetries")) { maxRetries = taskSpec["maxRetries"].GetInt(); }
|
||||||
uint8_t retryIntervalSeconds = 0;
|
uint8_t retryIntervalSeconds = 0;
|
||||||
if (taskSpec.HasMember("retryIntervalSeconds")) { retryIntervalSeconds = taskSpec["retryIntervalSeconds"].GetInt(); }
|
if (taskSpec.HasMember(
|
||||||
|
"retryIntervalSeconds")) { retryIntervalSeconds = taskSpec["retryIntervalSeconds"].GetInt(); }
|
||||||
|
|
||||||
// Children / parents
|
// Children / parents
|
||||||
std::vector<std::string> children;
|
std::vector<std::string> children;
|
||||||
|
|||||||
@@ -53,8 +53,7 @@ namespace daggy {
|
|||||||
TaskID taskID,
|
TaskID taskID,
|
||||||
const Task &task,
|
const Task &task,
|
||||||
executors::task::TaskExecutor &executor,
|
executors::task::TaskExecutor &executor,
|
||||||
loggers::dag_run::DAGLoggerBase & logger)
|
loggers::dag_run::DAGLoggerBase &logger) {
|
||||||
{
|
|
||||||
std::vector<AttemptRecord> attempts;
|
std::vector<AttemptRecord> attempts;
|
||||||
logger.updateTaskState(runID, taskID, loggers::dag_run::RunState::RUNNING);
|
logger.updateTaskState(runID, taskID, loggers::dag_run::RunState::RUNNING);
|
||||||
|
|
||||||
@@ -111,7 +110,9 @@ namespace daggy {
|
|||||||
auto tid = t.value();
|
auto tid = t.value();
|
||||||
TaskState tsk{
|
TaskState tsk{
|
||||||
.tid = tid,
|
.tid = tid,
|
||||||
.fut = tq->addTask([tid, runID, &tasks, &executor, &logger]() {return runTask(runID, tid, tasks[tid], executor, logger);}),
|
.fut = tq->addTask([tid, runID, &tasks, &executor, &logger]() {
|
||||||
|
return runTask(runID, tid, tasks[tid], executor, logger);
|
||||||
|
}),
|
||||||
.complete = false
|
.complete = false
|
||||||
};
|
};
|
||||||
taskStates.push_back(std::move(tsk));
|
taskStates.push_back(std::move(tsk));
|
||||||
|
|||||||
@@ -6,13 +6,15 @@ using namespace daggy::loggers::dag_run;
|
|||||||
|
|
||||||
namespace daggy {
|
namespace daggy {
|
||||||
inline const fs::path FileSystemLogger::getCurrentPath() const { return root_ / "current"; }
|
inline const fs::path FileSystemLogger::getCurrentPath() const { return root_ / "current"; }
|
||||||
|
|
||||||
inline const fs::path FileSystemLogger::getRunsRoot() const { return root_ / "runs"; }
|
inline const fs::path FileSystemLogger::getRunsRoot() const { return root_ / "runs"; }
|
||||||
inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const { return getRunsRoot() / std::to_string(runID); }
|
|
||||||
|
inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const {
|
||||||
|
return getRunsRoot() / std::to_string(runID);
|
||||||
|
}
|
||||||
|
|
||||||
FileSystemLogger::FileSystemLogger(fs::path root)
|
FileSystemLogger::FileSystemLogger(fs::path root)
|
||||||
: root_(root)
|
: root_(root), nextRunID_(0) {
|
||||||
, nextRunID_(0)
|
|
||||||
{
|
|
||||||
const std::vector<fs::path> reqPaths{root_, getCurrentPath(), getRunsRoot()};
|
const std::vector<fs::path> reqPaths{root_, getCurrentPath(), getRunsRoot()};
|
||||||
for (const auto &path : reqPaths) {
|
for (const auto &path : reqPaths) {
|
||||||
if (!fs::exists(path)) { fs::create_directory(path); }
|
if (!fs::exists(path)) { fs::create_directory(path); }
|
||||||
@@ -40,11 +42,15 @@ namespace daggy {
|
|||||||
|
|
||||||
// Init the directory
|
// Init the directory
|
||||||
}
|
}
|
||||||
|
|
||||||
void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) {}
|
void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) {}
|
||||||
|
|
||||||
void FileSystemLogger::logTaskAttempt(DAGRunID, size_t taskID, const AttemptRecord &attempt) {}
|
void FileSystemLogger::logTaskAttempt(DAGRunID, size_t taskID, const AttemptRecord &attempt) {}
|
||||||
|
|
||||||
void FileSystemLogger::updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) {}
|
void FileSystemLogger::updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) {}
|
||||||
|
|
||||||
// Querying
|
// Querying
|
||||||
std::vector<DAGRunSummary> FileSystemLogger::getDAGs(uint32_t stateMask) {}
|
std::vector<DAGRunSummary> FileSystemLogger::getDAGs(uint32_t stateMask) {}
|
||||||
|
|
||||||
DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunId) {}
|
DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunId) {}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ namespace daggy {
|
|||||||
DAGRunID StdOutLogger::startDAGRun(std::string name, const std::vector<Task> &tasks) {
|
DAGRunID StdOutLogger::startDAGRun(std::string name, const std::vector<Task> &tasks) {
|
||||||
std::lock_guard<std::mutex> lock(guard_);
|
std::lock_guard<std::mutex> lock(guard_);
|
||||||
size_t runID = nextRunID_++;
|
size_t runID = nextRunID_++;
|
||||||
std::cout << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size() << " tasks" << std::endl;
|
std::cout << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size()
|
||||||
|
<< " tasks" << std::endl;
|
||||||
return runID;
|
return runID;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -23,16 +24,19 @@ namespace daggy {
|
|||||||
void StdOutLogger::logTaskAttempt(DAGRunID dagRunID, size_t taskID, const AttemptRecord &attempt) {
|
void StdOutLogger::logTaskAttempt(DAGRunID dagRunID, size_t taskID, const AttemptRecord &attempt) {
|
||||||
std::lock_guard<std::mutex> lock(guard_);
|
std::lock_guard<std::mutex> lock(guard_);
|
||||||
const std::string &msg = attempt.rc == 0 ? attempt.output : attempt.error;
|
const std::string &msg = attempt.rc == 0 ? attempt.output : attempt.error;
|
||||||
std::cout << "Task Attempt (" << dagRunID << '/' << taskID << "): Ran with RC " << attempt.rc << ": " << msg << std::endl;
|
std::cout << "Task Attempt (" << dagRunID << '/' << taskID << "): Ran with RC " << attempt.rc << ": "
|
||||||
|
<< msg << std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void StdOutLogger::updateTaskState(DAGRunID dagRunID, TaskID taskID, RunState state) {
|
void StdOutLogger::updateTaskState(DAGRunID dagRunID, TaskID taskID, RunState state) {
|
||||||
std::lock_guard<std::mutex> lock(guard_);
|
std::lock_guard<std::mutex> lock(guard_);
|
||||||
std::cout << "Task State Change (" << dagRunID << '/' << taskID << "): " << magic_enum::enum_name(state) << std::endl;
|
std::cout << "Task State Change (" << dagRunID << '/' << taskID << "): " << magic_enum::enum_name(state)
|
||||||
|
<< std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Querying
|
// Querying
|
||||||
std::vector<DAGRunSummary> StdOutLogger::getDAGs(uint32_t stateMask) { return {}; }
|
std::vector<DAGRunSummary> StdOutLogger::getDAGs(uint32_t stateMask) { return {}; }
|
||||||
|
|
||||||
DAGRunRecord StdOutLogger::getDAGRun(DAGRunID dagRunId) { return {}; }
|
DAGRunRecord StdOutLogger::getDAGRun(DAGRunID dagRunId) { return {}; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user