Large re-organization to split daggyd away from the core libdaggy.
This paves the way for implementing daggys and other utilities. Squashed commit of the following: commit 1f77239ab3c9e44d190eef94531a39501c8c4dfe Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 16:25:02 2021 -0300 Adding README, stdout support for daggyd logging commit c2c237224e84a3be68aaa597ce98af1365e74a13 Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 16:10:29 2021 -0300 removing old daggyd commit cfea2baf61ca10c535801c5a391d2d525a1a2d04 Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 16:10:09 2021 -0300 Moving tests into their sub-project folders commit e41ca42069bea1db16dd76b6684a3f692fef6b15 Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 15:57:40 2021 -0300 Splitting out daggyd from libdaggy commit be97b146c1d2446f5c03cb78707e921f18c60bd8 Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 15:56:55 2021 -0300 Splitting out daggyd from libdaggy commit cb61e140e9d6d8832d61fb7037fd4c0ff6edad00 Author: Ian Roddis <gitlab@ie2r.com> Date: Mon Oct 18 15:49:47 2021 -0300 moving daggy to libdaggy
This commit is contained in:
142
libdaggy/src/Utilities.cpp
Normal file
142
libdaggy/src/Utilities.cpp
Normal file
@@ -0,0 +1,142 @@
|
||||
#include <daggy/Serialization.hpp>
|
||||
#include <daggy/Utilities.hpp>
|
||||
#include <future>
|
||||
#include <iomanip>
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
namespace daggy {
|
||||
std::string globalSub(std::string string, const std::string &pattern,
|
||||
const std::string &replacement)
|
||||
{
|
||||
size_t pos = string.find(pattern);
|
||||
while (pos != std::string::npos) {
|
||||
string.replace(pos, pattern.size(), replacement);
|
||||
pos = string.find(pattern, pos + replacement.size());
|
||||
}
|
||||
return string;
|
||||
}
|
||||
|
||||
std::vector<std::vector<std::string>> interpolateValues(
|
||||
const std::vector<std::string> &raw, const ConfigValues &values)
|
||||
{
|
||||
std::vector<std::vector<std::string>> cooked{{}};
|
||||
|
||||
for (const auto &part : raw) {
|
||||
std::vector<std::string> expandedPart{part};
|
||||
|
||||
// Find all values of parameters, and expand them
|
||||
for (const auto &[paramRaw, paramValue] : values) {
|
||||
std::string param = "{{" + paramRaw + "}}";
|
||||
auto pos = part.find(param);
|
||||
if (pos == std::string::npos)
|
||||
continue;
|
||||
std::vector<std::string> newExpandedPart;
|
||||
|
||||
if (std::holds_alternative<std::string>(paramValue)) {
|
||||
for (auto &cmd : expandedPart) {
|
||||
newExpandedPart.push_back(
|
||||
globalSub(cmd, param, std::get<std::string>(paramValue)));
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (const auto &val :
|
||||
std::get<std::vector<std::string>>(paramValue)) {
|
||||
for (const auto &cmd : expandedPart) {
|
||||
newExpandedPart.push_back(globalSub(cmd, param, val));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expandedPart.swap(newExpandedPart);
|
||||
}
|
||||
|
||||
std::vector<std::vector<std::string>> newCommands;
|
||||
for (const auto &newPart : expandedPart) {
|
||||
for (auto cmd : cooked) {
|
||||
cmd.push_back(newPart);
|
||||
newCommands.emplace_back(cmd);
|
||||
}
|
||||
}
|
||||
cooked.swap(newCommands);
|
||||
}
|
||||
return cooked;
|
||||
}
|
||||
|
||||
TaskSet expandTaskSet(const TaskSet &tasks,
|
||||
executors::task::TaskExecutor &executor,
|
||||
const ConfigValues &interpolatedValues)
|
||||
{
|
||||
// Expand the tasks first
|
||||
TaskSet newTaskSet;
|
||||
for (const auto &[baseName, task] : tasks) {
|
||||
executor.validateTaskParameters(task.job);
|
||||
const auto newJobs =
|
||||
executor.expandTaskParameters(task.job, interpolatedValues);
|
||||
size_t i = 0;
|
||||
for (const auto &newJob : newJobs) {
|
||||
Task newTask{task};
|
||||
newTask.job = newJob;
|
||||
newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask);
|
||||
++i;
|
||||
}
|
||||
}
|
||||
return newTaskSet;
|
||||
}
|
||||
|
||||
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks)
|
||||
{
|
||||
// Add the missing vertices
|
||||
for (const auto &[name, task] : tasks) {
|
||||
dag.addVertex(name, task);
|
||||
}
|
||||
|
||||
// Add edges
|
||||
for (const auto &[name, t] : tasks) {
|
||||
const auto &task = t;
|
||||
dag.addEdgeIf(name, [&task](const auto &v) {
|
||||
return task.children.count(v.data.definedName) > 0;
|
||||
});
|
||||
}
|
||||
|
||||
if (!dag.isValid()) {
|
||||
throw std::runtime_error("DAG contains a cycle");
|
||||
}
|
||||
}
|
||||
|
||||
TaskDAG buildDAGFromTasks(
|
||||
const TaskSet &tasks,
|
||||
const std::unordered_map<std::string,
|
||||
std::vector<loggers::dag_run::StateUpdateRecord>>
|
||||
&updates)
|
||||
{
|
||||
TaskDAG dag;
|
||||
updateDAGFromTasks(dag, tasks);
|
||||
|
||||
// Replay any updates
|
||||
for (const auto &[taskName, taskUpdates] : updates) {
|
||||
for (const auto &update : taskUpdates) {
|
||||
switch (update.state) {
|
||||
case RunState::RUNNING:
|
||||
case RunState::RETRY:
|
||||
case RunState::PAUSED:
|
||||
case RunState::ERRORED:
|
||||
case RunState::KILLED:
|
||||
dag.setVertexState(taskName, RunState::RUNNING);
|
||||
break;
|
||||
case RunState::COMPLETED:
|
||||
case RunState::QUEUED:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dag;
|
||||
}
|
||||
|
||||
std::ostream &operator<<(std::ostream &os, const TimePoint &tp)
|
||||
{
|
||||
os << tp.time_since_epoch().count() << std::endl;
|
||||
return os;
|
||||
}
|
||||
} // namespace daggy
|
||||
Reference in New Issue
Block a user