Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- Running project through clang-tidy formatter.

Browse Source
This commit is contained in:
Ian Roddis
2021-08-09 15:07:16 -03:00
parent 1fcdf66829
commit 30aea0818c
10 changed files with 129 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
daggy/include/daggy/Serialization.hpp
View File

@@ -15,9 +15,11 @@ namespace rj = rapidjson;
namespace daggy {
// Parameters
ParameterValues parametersFromJSON(const std::string &jsonSpec);
ParameterValues parametersFromJSON(const rj::Document &spec);
// Tasks
std::vector<Task> tasksFromJSON(const std::string &jsonSpec, const ParameterValues &parameters = {});
std::vector<Task> tasksFromJSON(const rj::Document &spec, const ParameterValues &parameters = {});
}

3
daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp
View File

@@ -8,8 +8,7 @@ namespace daggy {
class ForkingTaskExecutor : public TaskExecutor {
public:
ForkingTaskExecutor(size_t nThreads)
: TaskExecutor(nThreads)
{}
: TaskExecutor(nThreads) {}
const std::string getName() const override { return "ForkingTaskExecutor"; }

4
daggy/include/daggy/loggers/dag_run/DAGLoggerBase.hpp
View File

@@ -56,12 +56,16 @@ namespace daggy {
public:
// Execution
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) = 0;
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) = 0;
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) = 0;
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) = 0;
// Querying
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) = 0;
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId) = 0;
};
}

6
daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp
View File

@@ -40,12 +40,16 @@ namespace daggy {
// Execution
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
// Querying
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
private:
@@ -56,7 +60,9 @@ namespace daggy {
// std::unordered_map<fs::path, std::mutex> runLocks;
inline const fs::path getCurrentPath() const;
inline const fs::path getRunsRoot() const;
inline const fs::path getRunRoot(DAGRunID runID) const;
};
}

4
daggy/include/daggy/loggers/dag_run/StdOutLogger.hpp
View File

@@ -18,12 +18,16 @@ namespace daggy {
// Execution
virtual DAGRunID startDAGRun(std::string name, const std::vector<Task> &tasks) override;
virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) override;
virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) override;
virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) override;
// Querying
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
virtual DAGRunRecord getDAGRun(DAGRunID dagRunId);
private:

10
daggy/src/Serialization.cpp
View File

@@ -24,7 +24,9 @@ namespace daggy {
std::vector<std::string> values;
for (size_t i = 0; i < it->value.Size(); ++i) {
if (!it->value[i].IsString()) {
throw std::runtime_error("Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) + " is not a string.");
throw std::runtime_error(
"Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) +
" is not a string.");
}
values.emplace_back(it->value[i].GetString());
}
@@ -32,7 +34,8 @@ namespace daggy {
} else if (it->value.IsString()) {
parameters[name] = it->value.GetString();
} else {
throw std::runtime_error("Attribute for " + std::string{it->name.GetString()} + " is not a string or an array.");
throw std::runtime_error(
"Attribute for " + std::string{it->name.GetString()} + " is not a string or an array.");
}
}
return parameters;
@@ -77,7 +80,8 @@ namespace daggy {
uint8_t maxRetries = 0;
if (taskSpec.HasMember("maxRetries")) { maxRetries = taskSpec["maxRetries"].GetInt(); }
uint8_t retryIntervalSeconds = 0;
if (taskSpec.HasMember("retryIntervalSeconds")) { retryIntervalSeconds = taskSpec["retryIntervalSeconds"].GetInt(); }
if (taskSpec.HasMember(
"retryIntervalSeconds")) { retryIntervalSeconds = taskSpec["retryIntervalSeconds"].GetInt(); }
// Children / parents
std::vector<std::string> children;

7
daggy/src/Utilities.cpp
View File

@@ -53,8 +53,7 @@ namespace daggy {
TaskID taskID,
const Task &task,
executors::task::TaskExecutor &executor,
loggers::dag_run::DAGLoggerBase & logger)
{
loggers::dag_run::DAGLoggerBase &logger) {
std::vector<AttemptRecord> attempts;
logger.updateTaskState(runID, taskID, loggers::dag_run::RunState::RUNNING);
@@ -111,7 +110,9 @@ namespace daggy {
auto tid = t.value();
TaskState tsk{
.tid = tid,
.fut = tq->addTask([tid, runID, &tasks, &executor, &logger]() {return runTask(runID, tid, tasks[tid], executor, logger);}),
.fut = tq->addTask([tid, runID, &tasks, &executor, &logger]() {
return runTask(runID, tid, tasks[tid], executor, logger);
}),
.complete = false
};
taskStates.push_back(std::move(tsk));

14
daggy/src/loggers/dag_run/FileSystemLogger.cpp
View File

@@ -6,13 +6,15 @@ using namespace daggy::loggers::dag_run;
namespace daggy {
inline const fs::path FileSystemLogger::getCurrentPath() const { return root_ / "current"; }
inline const fs::path FileSystemLogger::getRunsRoot() const { return root_ / "runs"; }
inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const { return getRunsRoot() / std::to_string(runID); }
inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const {
return getRunsRoot() / std::to_string(runID);
}
FileSystemLogger::FileSystemLogger(fs::path root)
: root_(root)
, nextRunID_(0)
{
: root_(root), nextRunID_(0) {
const std::vector<fs::path> reqPaths{root_, getCurrentPath(), getRunsRoot()};
for (const auto &path : reqPaths) {
if (!fs::exists(path)) { fs::create_directory(path); }
@@ -40,11 +42,15 @@ namespace daggy {
// Init the directory
}
void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) {}
void FileSystemLogger::logTaskAttempt(DAGRunID, size_t taskID, const AttemptRecord &attempt) {}
void FileSystemLogger::updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) {}
// Querying
std::vector<DAGRunSummary> FileSystemLogger::getDAGs(uint32_t stateMask) {}
DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunId) {}
}

10
daggy/src/loggers/dag_run/StdOutLogger.cpp
View File

@@ -11,7 +11,8 @@ namespace daggy {
DAGRunID StdOutLogger::startDAGRun(std::string name, const std::vector<Task> &tasks) {
std::lock_guard<std::mutex> lock(guard_);
size_t runID = nextRunID_++;
std::cout << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size() << " tasks" << std::endl;
std::cout << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size()
<< " tasks" << std::endl;
return runID;
}
@@ -23,16 +24,19 @@ namespace daggy {
void StdOutLogger::logTaskAttempt(DAGRunID dagRunID, size_t taskID, const AttemptRecord &attempt) {
std::lock_guard<std::mutex> lock(guard_);
const std::string &msg = attempt.rc == 0 ? attempt.output : attempt.error;
std::cout << "Task Attempt (" << dagRunID << '/' << taskID << "): Ran with RC " << attempt.rc << ": " << msg << std::endl;
std::cout << "Task Attempt (" << dagRunID << '/' << taskID << "): Ran with RC " << attempt.rc << ": "
<< msg << std::endl;
}
void StdOutLogger::updateTaskState(DAGRunID dagRunID, TaskID taskID, RunState state) {
std::lock_guard<std::mutex> lock(guard_);
std::cout << "Task State Change (" << dagRunID << '/' << taskID << "): " << magic_enum::enum_name(state) << std::endl;
std::cout << "Task State Change (" << dagRunID << '/' << taskID << "): " << magic_enum::enum_name(state)
<< std::endl;
}
// Querying
std::vector<DAGRunSummary> StdOutLogger::getDAGs(uint32_t stateMask) { return {}; }
DAGRunRecord StdOutLogger::getDAGRun(DAGRunID dagRunId) { return {}; }
}
}