Large re-organization to split daggyd away from the core libdaggy.

This paves the way for implementing daggys and other utilities.

Squashed commit of the following:

commit 1f77239ab3c9e44d190eef94531a39501c8c4dfe
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 16:25:02 2021 -0300

    Adding README, stdout support for daggyd logging

commit c2c237224e84a3be68aaa597ce98af1365e74a13
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 16:10:29 2021 -0300

    removing old daggyd

commit cfea2baf61ca10c535801c5a391d2d525a1a2d04
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 16:10:09 2021 -0300

    Moving tests into their sub-project folders

commit e41ca42069bea1db16dd76b6684a3f692fef6b15
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 15:57:40 2021 -0300

    Splitting out daggyd from libdaggy

commit be97b146c1d2446f5c03cb78707e921f18c60bd8
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 15:56:55 2021 -0300

    Splitting out daggyd from libdaggy

commit cb61e140e9d6d8832d61fb7037fd4c0ff6edad00
Author: Ian Roddis <gitlab@ie2r.com>
Date:   Mon Oct 18 15:49:47 2021 -0300

    moving daggy to libdaggy
This commit is contained in:
Ian Roddis
2021-10-18 16:28:40 -03:00
parent 612bc8af8a
commit 470a6f2bb7
59 changed files with 586 additions and 52 deletions

142
libdaggy/src/Utilities.cpp Normal file
View File

@@ -0,0 +1,142 @@
#include <daggy/Serialization.hpp>
#include <daggy/Utilities.hpp>
#include <future>
#include <iomanip>
using namespace std::chrono_literals;
namespace daggy {
std::string globalSub(std::string string, const std::string &pattern,
const std::string &replacement)
{
size_t pos = string.find(pattern);
while (pos != std::string::npos) {
string.replace(pos, pattern.size(), replacement);
pos = string.find(pattern, pos + replacement.size());
}
return string;
}
std::vector<std::vector<std::string>> interpolateValues(
const std::vector<std::string> &raw, const ConfigValues &values)
{
std::vector<std::vector<std::string>> cooked{{}};
for (const auto &part : raw) {
std::vector<std::string> expandedPart{part};
// Find all values of parameters, and expand them
for (const auto &[paramRaw, paramValue] : values) {
std::string param = "{{" + paramRaw + "}}";
auto pos = part.find(param);
if (pos == std::string::npos)
continue;
std::vector<std::string> newExpandedPart;
if (std::holds_alternative<std::string>(paramValue)) {
for (auto &cmd : expandedPart) {
newExpandedPart.push_back(
globalSub(cmd, param, std::get<std::string>(paramValue)));
}
}
else {
for (const auto &val :
std::get<std::vector<std::string>>(paramValue)) {
for (const auto &cmd : expandedPart) {
newExpandedPart.push_back(globalSub(cmd, param, val));
}
}
}
expandedPart.swap(newExpandedPart);
}
std::vector<std::vector<std::string>> newCommands;
for (const auto &newPart : expandedPart) {
for (auto cmd : cooked) {
cmd.push_back(newPart);
newCommands.emplace_back(cmd);
}
}
cooked.swap(newCommands);
}
return cooked;
}
TaskSet expandTaskSet(const TaskSet &tasks,
executors::task::TaskExecutor &executor,
const ConfigValues &interpolatedValues)
{
// Expand the tasks first
TaskSet newTaskSet;
for (const auto &[baseName, task] : tasks) {
executor.validateTaskParameters(task.job);
const auto newJobs =
executor.expandTaskParameters(task.job, interpolatedValues);
size_t i = 0;
for (const auto &newJob : newJobs) {
Task newTask{task};
newTask.job = newJob;
newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask);
++i;
}
}
return newTaskSet;
}
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks)
{
// Add the missing vertices
for (const auto &[name, task] : tasks) {
dag.addVertex(name, task);
}
// Add edges
for (const auto &[name, t] : tasks) {
const auto &task = t;
dag.addEdgeIf(name, [&task](const auto &v) {
return task.children.count(v.data.definedName) > 0;
});
}
if (!dag.isValid()) {
throw std::runtime_error("DAG contains a cycle");
}
}
TaskDAG buildDAGFromTasks(
const TaskSet &tasks,
const std::unordered_map<std::string,
std::vector<loggers::dag_run::StateUpdateRecord>>
&updates)
{
TaskDAG dag;
updateDAGFromTasks(dag, tasks);
// Replay any updates
for (const auto &[taskName, taskUpdates] : updates) {
for (const auto &update : taskUpdates) {
switch (update.state) {
case RunState::RUNNING:
case RunState::RETRY:
case RunState::PAUSED:
case RunState::ERRORED:
case RunState::KILLED:
dag.setVertexState(taskName, RunState::RUNNING);
break;
case RunState::COMPLETED:
case RunState::QUEUED:
break;
}
}
}
return dag;
}
std::ostream &operator<<(std::ostream &os, const TimePoint &tp)
{
os << tp.time_since_epoch().count() << std::endl;
return os;
}
} // namespace daggy