Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding clang-format, and reformating all sourcecode

Browse Source
This commit is contained in:
Ian Roddis
2021-09-21 09:41:11 -03:00
parent 39d5ae08be
commit 288ce28d29
36 changed files with 3355 additions and 2802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
.clang-format Normal file
View File

@@ -0,0 +1,198 @@
---
DisableFormat: false
Language: Cpp
Standard: Auto
# Indentation rules
IndentWidth: 2
AccessModifierOffset: -2
ConstructorInitializerIndentWidth: 2
ContinuationIndentWidth: 4
IndentCaseLabels: true
IndentGotoLabels: true
IndentPPDirectives: None
IndentWrappedFunctionNames: false
NamespaceIndentation: All
UseTab: Never
TabWidth: 8
# Brace wrapping rules
BreakBeforeBraces: Custom
BraceWrapping:
AfterEnum: true
AfterClass: true
AfterStruct: true
AfterUnion: true
AfterNamespace: false
AfterExternBlock: false
AfterCaseLabel: false
AfterControlStatement: false
AfterFunction: true
BeforeCatch: true
BeforeElse: true
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
# Line break rules
DeriveLineEnding: true
UseCRLF: false
KeepEmptyLinesAtTheStartOfBlocks: false
MaxEmptyLinesToKeep: 1
BinPackArguments: true
BinPackParameters: true
ExperimentalAutoDetectBinPacking: false
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: Yes
BreakInheritanceList: BeforeComma
BreakConstructorInitializers: BeforeComma
BreakBeforeInheritanceComma: true
BreakConstructorInitializersBeforeComma: true
BreakBeforeBinaryOperators: None
BreakBeforeTernaryOperators: true
BreakStringLiterals: true
AllowAllArgumentsOnNextLine: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowAllConstructorInitializersOnNextLine: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
AllowShortBlocksOnASingleLine: Never
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
AllowShortLambdasOnASingleLine: All
AllowShortLoopsOnASingleLine: false
# Line length rules
ColumnLimit: 80
ReflowComments: true
## line length penalties
## these determine where line breaks are inserted when over ColumnLimit
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyBreakTemplateDeclaration: 10
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
# Alignment rules
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignConsecutiveMacros: false
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: true
DerivePointerAlignment: true
PointerAlignment: Left
# Include ordering rules
IncludeBlocks: Regroup
SortIncludes: true
IncludeIsMainRegex: '([-_](test|unittest))?$'
IncludeIsMainSourceRegex: ''
IncludeCategories:
- Regex: '^".*\.h"'
Priority: 2
SortPriority: 0
- Regex: '^<.*\.h>'
Priority: 1
SortPriority: 0
- Regex: '^<.*'
Priority: 2
SortPriority: 0
- Regex: '.*'
Priority: 3
SortPriority: 0
# Namespace rules
CompactNamespaces: true
FixNamespaceComments: true
# Language extention macros
CommentPragmas: '^ IWYU pragma:'
MacroBlockBegin: ''
MacroBlockEnd: ''
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
StatementMacros:
- Q_UNUSED
- QT_REQUIRE_VERSION
# Spacing rules
SpaceAfterCStyleCast: false
SpaceAfterLogicalNot: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceBeforeSquareBrackets: false
SpaceInEmptyBlock: false
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
SpacesInConditionalStatement: false
SpacesInContainerLiterals: true
SpacesInParentheses: false
SpacesInSquareBrackets: false
# Rules for detecting embedded code blocks
RawStringFormats:
- Language: Cpp
Delimiters:
- cc
- CC
- cpp
- Cpp
- CPP
- 'c++'
- 'C++'
CanonicalDelimiter: ''
BasedOnStyle: google
- Language: TextProto
Delimiters:
- pb
- PB
- proto
- PROTO
EnclosingFunctions:
- EqualsProto
- EquivToProto
- PARSE_PARTIAL_TEXT_PROTO
- PARSE_TEST_PROTO
- PARSE_TEXT_PROTO
- ParseTextOrDie
- ParseTextProtoOrDie
CanonicalDelimiter: ''
BasedOnStyle: google
# C++ specific rules
Cpp11BracedListStyle: true
SortUsingDeclarations: true
...

92
daggy/include/daggy/DAG.hpp
View File

@@ -1,15 +1,15 @@
#pragma once #pragma once
#include <iostream>
#include <deque> #include <deque>
#include <functional>
#include <iostream>
#include <iterator>
#include <optional>
#include <queue>
#include <sstream>
#include <stdexcept> #include <stdexcept>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <iterator>
#include <functional>
#include <optional>
#include <sstream>
#include <queue>
#include "Defines.hpp" #include "Defines.hpp"
@@ -20,63 +20,67 @@
namespace daggy { namespace daggy {
template<typename K, typename V> template <typename K, typename V>
struct Vertex { struct Vertex
RunState state; {
uint32_t depCount; RunState state;
V data; uint32_t depCount;
std::unordered_set<K> children; V data;
}; std::unordered_set<K> children;
};
template <typename K, typename V>
class DAG
{
using Edge = std::pair<K, K>;
template<typename K, typename V> public:
class DAG { // Vertices
using Edge = std::pair<K, K>; void addVertex(K id, V data);
public:
// Vertices
void addVertex(K id, V data);
std::unordered_set<K> getVertices() const; std::unordered_set<K> getVertices() const;
// Edges // Edges
void addEdge(const K &src, const K &dst); void addEdge(const K &src, const K &dst);
void addEdgeIf(const K &src, std::function<bool(const Vertex<K, V> &v)> predicate); void addEdgeIf(const K &src,
std::function<bool(const Vertex<K, V> &v)> predicate);
bool isValid() const; bool isValid() const;
bool hasVertex(const K &from); bool hasVertex(const K &from);
const std::vector<Edge> &getEdges(); const std::vector<Edge> &getEdges();
// Attributes // Attributes
size_t size() const; size_t size() const;
bool empty() const; bool empty() const;
// Reset the DAG to completely unvisited // Reset the DAG to completely unvisited
void reset(); void reset();
// Reset any vertex with RUNNING state to QUEUED // Reset any vertex with RUNNING state to QUEUED
void resetRunning(); void resetRunning();
RunState getVertexState(const K &id) const; RunState getVertexState(const K &id) const;
void setVertexState(const K &id, RunState state); void setVertexState(const K &id, RunState state);
void forEach(std::function<void(const std::pair<K, Vertex<K, V>> &)> fun) const; void forEach(
std::function<void(const std::pair<K, Vertex<K, V>> &)> fun) const;
bool allVisited() const; bool allVisited() const;
std::optional<std::pair<K, V>> visitNext(); std::optional<std::pair<K, V>> visitNext();
Vertex<K, V> &getVertex(const K &id); Vertex<K, V> &getVertex(const K &id);
void completeVisit(const K &id); void completeVisit(const K &id);
private: private:
std::unordered_map<K, Vertex<K, V>> vertices_; std::unordered_map<K, Vertex<K, V>> vertices_;
}; };
} } // namespace daggy
#include "DAG.impl.hxx" #include "DAG.impl.hxx"

285
daggy/include/daggy/DAG.impl.hxx
View File

@@ -1,148 +1,183 @@
namespace daggy { namespace daggy {
template<typename K, typename V> template <typename K, typename V>
size_t DAG<K, V>::size() const { return vertices_.size(); } size_t DAG<K, V>::size() const
{
return vertices_.size();
}
template<typename K, typename V> template <typename K, typename V>
bool DAG<K, V>::empty() const { return vertices_.empty(); } bool DAG<K, V>::empty() const
{
return vertices_.empty();
}
template<typename K, typename V> template <typename K, typename V>
bool DAG<K, V>::hasVertex(const K &id) { return vertices_.count(id) != 0; } bool DAG<K, V>::hasVertex(const K &id)
{
return vertices_.count(id) != 0;
}
template<typename K, typename V> template <typename K, typename V>
Vertex <K, V> &DAG<K, V>::getVertex(const K &id) { return vertices_.at(id); } Vertex<K, V> &DAG<K, V>::getVertex(const K &id)
{
return vertices_.at(id);
}
template<typename K, typename V> template <typename K, typename V>
std::unordered_set<K> DAG<K, V>::getVertices() const { std::unordered_set<K> DAG<K, V>::getVertices() const
std::unordered_set<K> keys; {
for (const auto it : vertices_) { std::unordered_set<K> keys;
keys.insert(it.first); for (const auto it : vertices_) {
} keys.insert(it.first);
return keys; }
return keys;
}
template <typename K, typename V>
void DAG<K, V>::addVertex(K id, V data)
{
if (vertices_.count(id) != 0) {
std::stringstream ss;
ss << "A vertex with ID " << id << " already exists in the DAG";
throw std::runtime_error(ss.str());
}
vertices_.emplace(
id,
Vertex<K, V>{.state = RunState::QUEUED, .depCount = 0, .data = data});
}
template <typename K, typename V>
void DAG<K, V>::addEdge(const K &from, const K &to)
{
if (vertices_.find(from) == vertices_.end())
throw std::runtime_error("No such vertex");
if (vertices_.find(to) == vertices_.end())
throw std::runtime_error("No such vertex");
vertices_.at(from).children.insert(to);
vertices_.at(to).depCount++;
}
template <typename K, typename V>
void DAG<K, V>::addEdgeIf(
const K &src, std::function<bool(const Vertex<K, V> &v)> predicate)
{
auto &parent = vertices_.at(src);
for (auto &[name, vertex] : vertices_) {
if (!predicate(vertex))
continue;
if (name == src)
continue;
parent.children.insert(name);
vertex.depCount++;
}
}
template <typename K, typename V>
bool DAG<K, V>::isValid() const
{
std::unordered_map<K, size_t> depCounts;
std::queue<K> ready;
size_t processed = 0;
for (const auto &[k, v] : vertices_) {
depCounts[k] = v.depCount;
if (v.depCount == 0)
ready.push(k);
} }
template<typename K, typename V> while (!ready.empty()) {
void DAG<K, V>::addVertex(K id, V data) { const auto &k = ready.front();
if (vertices_.count(id) != 0) { for (const auto &child : vertices_.at(k).children) {
std::stringstream ss; auto dc = --depCounts[child];
ss << "A vertex with ID " << id << " already exists in the DAG"; if (dc == 0)
throw std::runtime_error(ss.str()); ready.push(child);
} }
vertices_.emplace(id, Vertex<K, V>{.state = RunState::QUEUED, .depCount = 0, .data = data}); processed++;
ready.pop();
} }
template<typename K, typename V> return processed == vertices_.size();
void DAG<K, V>::addEdge(const K &from, const K &to) { }
if (vertices_.find(from) == vertices_.end()) throw std::runtime_error("No such vertex");
if (vertices_.find(to) == vertices_.end()) throw std::runtime_error("No such vertex"); template <typename K, typename V>
vertices_.at(from).children.insert(to); void DAG<K, V>::reset()
vertices_.at(to).depCount++; {
// Reset the state of all vertices
for (auto &[_, v] : vertices_) {
v.state = RunState::QUEUED;
v.depCount = 0;
} }
template<typename K, typename V> // Calculate the upstream count
void DAG<K, V>::addEdgeIf(const K &src, std::function<bool(const Vertex <K, V> &v)> predicate) { for (auto &[_, v] : vertices_) {
auto & parent = vertices_.at(src); for (auto c : v.children) {
for (auto &[name, vertex]: vertices_) { vertices_.at(c).depCount++;
if (! predicate(vertex)) continue; }
if (name == src) continue;
parent.children.insert(name);
vertex.depCount++;
}
} }
}
template<typename K, typename V> template <typename K, typename V>
bool DAG<K, V>::isValid() const { void DAG<K, V>::resetRunning()
std::unordered_map<K, size_t> depCounts; {
std::queue<K> ready; for (auto &[k, v] : vertices_) {
size_t processed = 0; if (v.state != +RunState::RUNNING)
continue;
for (const auto & [k, v] : vertices_) { v.state = RunState::QUEUED;
depCounts[k] = v.depCount;
if (v.depCount == 0) ready.push(k);
}
while (! ready.empty()) {
const auto & k = ready.front();
for (const auto & child : vertices_.at(k).children) {
auto dc = --depCounts[child];
if (dc == 0) ready.push(child);
}
processed++;
ready.pop();
}
return processed == vertices_.size();
} }
}
template<typename K, typename V> template <typename K, typename V>
void DAG<K, V>::reset() { void DAG<K, V>::setVertexState(const K &id, RunState state)
// Reset the state of all vertices {
for (auto &[_, v]: vertices_) { vertices_.at(id).state = state;
v.state = RunState::QUEUED; }
v.depCount = 0;
}
// Calculate the upstream count template <typename K, typename V>
for (auto &[_, v]: vertices_) { bool DAG<K, V>::allVisited() const
for (auto c: v.children) { {
vertices_.at(c).depCount++; for (const auto &[_, v] : vertices_) {
} if (v.state != +RunState::COMPLETED)
} return false;
} }
return true;
}
template<typename K, typename V> template <typename K, typename V>
void DAG<K, V>::resetRunning() { std::optional<std::pair<K, V>> DAG<K, V>::visitNext()
for (auto &[k, v]: vertices_) { {
if (v.state != +RunState::RUNNING) continue; for (auto &[k, v] : vertices_) {
v.state = RunState::QUEUED; if (v.state != +RunState::QUEUED)
} continue;
if (v.depCount != 0)
continue;
v.state = RunState::RUNNING;
return std::make_pair(k, v.data);
} }
return {};
}
template<typename K, typename V> template <typename K, typename V>
void DAG<K, V>::setVertexState(const K &id, RunState state) { void DAG<K, V>::completeVisit(const K &id)
vertices_.at(id).state = state; {
auto &v = vertices_.at(id);
v.state = RunState::COMPLETED;
for (auto c : v.children) {
--vertices_.at(c).depCount;
} }
}
template<typename K, typename V> template <typename K, typename V>
bool DAG<K, V>::allVisited() const { void DAG<K, V>::forEach(std::function<void(const std::pair<K, Vertex<K, V>> &)
for (const auto &[_, v]: vertices_) {
if (v.state != +RunState::COMPLETED) return false; >
} fun) const
return true; {
for (auto it = vertices_.begin(); it != vertices_.
end();
++it) {
fun(*it);
} }
}
template<typename K, typename V> } // namespace daggy
std::optional<std::pair<K, V>>
DAG<K, V>::visitNext() {
for (auto &[k, v]: vertices_) {
if (v.state != +RunState::QUEUED) continue;
if (v.depCount != 0) continue;
v.state = RunState::RUNNING;
return std::make_pair(k, v.data);
}
return {};
}
template<typename K, typename V>
void DAG<K, V>::completeVisit(const K &id) {
auto &v = vertices_.at(id);
v.state = RunState::COMPLETED;
for (auto c: v.children) {
--vertices_.at(c).depCount;
}
}
template<typename K, typename V>
void DAG<K, V>::forEach(std::function<void(const std::pair<K, Vertex < K, V>> &)
> fun) const {
for (
auto it = vertices_.begin();
it != vertices_.
end();
++it) {
fun(*it);
}
}
}

92
daggy/include/daggy/Defines.hpp
View File

@@ -1,5 +1,7 @@
#pragma once #pragma once
#include <enum.h>
#include <chrono> #include <chrono>
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
@@ -7,60 +9,56 @@
#include <variant> #include <variant>
#include <vector> #include <vector>
#include <enum.h>
namespace daggy { namespace daggy {
// Commands and parameters // Commands and parameters
using ConfigValue = std::variant<std::string, std::vector<std::string>>; using ConfigValue = std::variant<std::string, std::vector<std::string>>;
using ConfigValues = std::unordered_map<std::string, ConfigValue>; using ConfigValues = std::unordered_map<std::string, ConfigValue>;
using Command = std::vector<std::string>; using Command = std::vector<std::string>;
// Time // Time
using Clock = std::chrono::high_resolution_clock; using Clock = std::chrono::high_resolution_clock;
using TimePoint = std::chrono::time_point<Clock>; using TimePoint = std::chrono::time_point<Clock>;
// DAG Runs // DAG Runs
using DAGRunID = size_t; using DAGRunID = size_t;
BETTER_ENUM(RunState, uint32_t, BETTER_ENUM(RunState, uint32_t, QUEUED = 1 << 0, RUNNING = 1 << 1,
QUEUED = 1 << 0, RETRY = 1 << 2, ERRORED = 1 << 3, KILLED = 1 << 4,
RUNNING = 1 << 1, COMPLETED = 1 << 5);
RETRY = 1 << 2,
ERRORED = 1 << 3,
KILLED = 1 << 4,
COMPLETED = 1 << 5
);
struct Task { struct Task
std::string definedName; {
bool isGenerator; // True if the output of this task is a JSON set of tasks to complete std::string definedName;
uint32_t maxRetries; bool isGenerator; // True if the output of this task is a JSON set of tasks
uint32_t retryIntervalSeconds; // Time to wait between retries // to complete
ConfigValues job; // It's up to the individual inspectors to convert values from strings // array of strings uint32_t maxRetries;
std::unordered_set<std::string> children; uint32_t retryIntervalSeconds; // Time to wait between retries
std::unordered_set<std::string> parents; ConfigValues job; // It's up to the individual inspectors to convert values
// from strings // array of strings
std::unordered_set<std::string> children;
std::unordered_set<std::string> parents;
bool operator==(const Task &other) const { bool operator==(const Task &other) const
return (definedName == other.definedName) {
and (maxRetries == other.maxRetries) return (definedName == other.definedName) and
and (retryIntervalSeconds == other.retryIntervalSeconds) (maxRetries == other.maxRetries) and
and (job == other.job) (retryIntervalSeconds == other.retryIntervalSeconds) and
and (children == other.children) (job == other.job) and (children == other.children) and
and (parents == other.parents) (parents == other.parents) and (isGenerator == other.isGenerator);
and (isGenerator == other.isGenerator); }
} };
};
using TaskSet = std::unordered_map<std::string, Task>; using TaskSet = std::unordered_map<std::string, Task>;
struct AttemptRecord { struct AttemptRecord
TimePoint startTime; {
TimePoint stopTime; TimePoint startTime;
int rc; // RC from the task TimePoint stopTime;
std::string executorLog; // Logs from the dag_executor int rc; // RC from the task
std::string outputLog; // stdout from command std::string executorLog; // Logs from the dag_executor
std::string errorLog; // stderr from command std::string outputLog; // stdout from command
}; std::string errorLog; // stderr from command
} };
} // namespace daggy
BETTER_ENUMS_DECLARE_STD_HASH(daggy::RunState) BETTER_ENUMS_DECLARE_STD_HASH(daggy::RunState)

51
daggy/include/daggy/Serialization.hpp
View File

@@ -1,45 +1,48 @@
#pragma once #pragma once
#include <vector>
#include <string>
#include <variant>
#include <unordered_map>
#include <rapidjson/document.h> #include <rapidjson/document.h>
#include <string>
#include <unordered_map>
#include <variant>
#include <vector>
#include "Defines.hpp" #include "Defines.hpp"
namespace rj = rapidjson; namespace rj = rapidjson;
namespace daggy { namespace daggy {
void checkRJParse(const rj::ParseResult &result, const std::string &prefix = ""); void checkRJParse(const rj::ParseResult &result,
const std::string &prefix = "");
// Parameters // Parameters
ConfigValues configFromJSON(const std::string &jsonSpec); ConfigValues configFromJSON(const std::string &jsonSpec);
ConfigValues configFromJSON(const rj::Value &spec); ConfigValues configFromJSON(const rj::Value &spec);
std::string configToJSON(const ConfigValues &config); std::string configToJSON(const ConfigValues &config);
// Tasks // Tasks
Task Task taskFromJSON(const std::string &name, const rj::Value &spec,
taskFromJSON(const std::string &name, const rj::Value &spec, const ConfigValues &jobDefaults = {}); const ConfigValues &jobDefaults = {});
TaskSet tasksFromJSON(const std::string &jsonSpec, const ConfigValues &jobDefaults = {}); TaskSet tasksFromJSON(const std::string &jsonSpec,
const ConfigValues &jobDefaults = {});
TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults = {}); TaskSet tasksFromJSON(const rj::Value &spec,
const ConfigValues &jobDefaults = {});
std::string taskToJSON(const Task &task); std::string taskToJSON(const Task &task);
std::string tasksToJSON(const TaskSet &tasks); std::string tasksToJSON(const TaskSet &tasks);
// Attempt Records // Attempt Records
std::string attemptRecordToJSON(const AttemptRecord &attemptRecord); std::string attemptRecordToJSON(const AttemptRecord &attemptRecord);
// default serialization // default serialization
std::ostream &operator<<(std::ostream &os, const Task &task); std::ostream &operator<<(std::ostream &os, const Task &task);
std::string timePointToString(const TimePoint &tp); std::string timePointToString(const TimePoint &tp);
TimePoint stringToTimePoint(const std::string &timeStr); TimePoint stringToTimePoint(const std::string &timeStr);
} } // namespace daggy

74
daggy/include/daggy/Server.hpp
View File

@@ -1,58 +1,68 @@
#pragma once #pragma once
#include <filesystem>
#include <pistache/description.h> #include <pistache/description.h>
#include <pistache/endpoint.h> #include <pistache/endpoint.h>
#include <pistache/http.h> #include <pistache/http.h>
#include <filesystem>
#include "ThreadPool.hpp" #include "ThreadPool.hpp"
#include "loggers/dag_run/DAGRunLogger.hpp"
#include "executors/task/TaskExecutor.hpp" #include "executors/task/TaskExecutor.hpp"
#include "loggers/dag_run/DAGRunLogger.hpp"
namespace fs = std::filesystem; namespace fs = std::filesystem;
namespace daggy { namespace daggy {
class Server { class Server
public: {
Server(const Pistache::Address &listenSpec, loggers::dag_run::DAGRunLogger &logger, public:
executors::task::TaskExecutor &executor, Server(const Pistache::Address &listenSpec,
size_t nDAGRunners loggers::dag_run::DAGRunLogger &logger,
) executors::task::TaskExecutor &executor, size_t nDAGRunners)
: endpoint_(listenSpec), desc_("Daggy API", "0.1"), logger_(logger), executor_(executor), : endpoint_(listenSpec)
runnerPool_(nDAGRunners) {} , desc_("Daggy API", "0.1")
, logger_(logger)
, executor_(executor)
, runnerPool_(nDAGRunners)
{
}
Server &setWebHandlerThreads(size_t nThreads); Server &setWebHandlerThreads(size_t nThreads);
Server &setSSLCertificates(const fs::path &cert, const fs::path &key); Server &setSSLCertificates(const fs::path &cert, const fs::path &key);
void init(int threads = 1); void init(int threads = 1);
void start(); void start();
uint16_t getPort() const; uint16_t getPort() const;
void shutdown(); void shutdown();
private: private:
void createDescription(); void createDescription();
void handleRunDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void handleRunDAG(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response);
void handleGetDAGRuns(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void handleGetDAGRuns(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response);
void handleGetDAGRun(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void handleGetDAGRun(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response);
void handleReady(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); void handleReady(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response);
bool handleAuth(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response); bool handleAuth(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter &response);
Pistache::Http::Endpoint endpoint_; Pistache::Http::Endpoint endpoint_;
Pistache::Rest::Description desc_; Pistache::Rest::Description desc_;
Pistache::Rest::Router router_; Pistache::Rest::Router router_;
loggers::dag_run::DAGRunLogger &logger_; loggers::dag_run::DAGRunLogger &logger_;
executors::task::TaskExecutor &executor_; executors::task::TaskExecutor &executor_;
ThreadPool runnerPool_; ThreadPool runnerPool_;
}; };
} } // namespace daggy

307
daggy/include/daggy/ThreadPool.hpp
View File

@@ -1,165 +1,186 @@
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <condition_variable>
#include <functional>
#include <future>
#include <list>
#include <memory>
#include <queue>
#include <thread> #include <thread>
#include <vector> #include <vector>
#include <memory>
#include <condition_variable>
#include <future>
#include <queue>
#include <functional>
#include <list>
using namespace std::chrono_literals; using namespace std::chrono_literals;
namespace daggy { namespace daggy {
/* /*
A Task Queue is a collection of async tasks to be executed by the A Task Queue is a collection of async tasks to be executed by the
thread pool. Using individual task queues allows for a rough QoS thread pool. Using individual task queues allows for a rough QoS
when a single thread may be submitting batches of requests -- when a single thread may be submitting batches of requests --
one producer won't starve out another, but all tasks will be run one producer won't starve out another, but all tasks will be run
as quickly as possible. as quickly as possible.
*/ */
class TaskQueue { class TaskQueue
public: {
template<class F, class... Args> public:
decltype(auto) addTask(F &&f, Args &&... args) { template <class F, class... Args>
// using return_type = std::invoke_result<F, Args...>::type; decltype(auto) addTask(F &&f, Args &&...args)
using return_type = std::invoke_result_t<F, Args...>; {
// using return_type = std::invoke_result<F, Args...>::type;
using return_type = std::invoke_result_t<F, Args...>;
std::packaged_task<return_type()> task( std::packaged_task<return_type()> task(
std::bind(std::forward<F>(f), std::forward<Args>(args)...) std::bind(std::forward<F>(f), std::forward<Args>(args)...));
);
std::future<return_type> res = task.get_future(); std::future<return_type> res = task.get_future();
{
std::lock_guard<std::mutex> guard(mtx_);
tasks_.emplace(std::move(task));
}
return res;
}
std::packaged_task<void()> pop()
{
std::lock_guard<std::mutex> guard(mtx_);
auto task = std::move(tasks_.front());
tasks_.pop();
return task;
}
size_t size()
{
std::lock_guard<std::mutex> guard(mtx_);
return tasks_.size();
}
bool empty()
{
std::lock_guard<std::mutex> guard(mtx_);
return tasks_.empty();
}
private:
std::queue<std::packaged_task<void()>> tasks_;
std::mutex mtx_;
};
class ThreadPool
{
public:
explicit ThreadPool(size_t nWorkers)
: tqit_(taskQueues_.begin())
, stop_(false)
, drain_(false)
{
resize(nWorkers);
}
~ThreadPool()
{
shutdown();
}
void shutdown()
{
stop_ = true;
cv_.notify_all();
for (std::thread &worker : workers_) {
if (worker.joinable())
worker.join();
}
}
void drain()
{
drain_ = true;
while (true) {
{
std::lock_guard<std::mutex> guard(mtx_);
if (taskQueues_.empty())
break;
}
std::this_thread::sleep_for(250ms);
}
}
void restart()
{
drain_ = false;
}
void resize(size_t nWorkers)
{
shutdown();
workers_.clear();
stop_ = false;
for (size_t i = 0; i < nWorkers; ++i)
workers_.emplace_back([&] {
while (true) {
std::packaged_task<void()> task;
{ {
std::lock_guard<std::mutex> guard(mtx_); std::unique_lock<std::mutex> lock(mtx_);
tasks_.emplace(std::move(task)); cv_.wait(lock, [&] { return stop_ || !taskQueues_.empty(); });
if (taskQueues_.empty()) {
if (stop_)
return;
continue;
}
if (tqit_ == taskQueues_.end())
tqit_ = taskQueues_.begin();
task = std::move((*tqit_)->pop());
if ((*tqit_)->empty()) {
tqit_ = taskQueues_.erase(tqit_);
}
else {
tqit_++;
}
} }
return res; task();
} }
});
std::packaged_task<void()> pop() {
std::lock_guard<std::mutex> guard(mtx_);
auto task = std::move(tasks_.front());
tasks_.pop();
return task;
}
size_t size() {
std::lock_guard<std::mutex> guard(mtx_);
return tasks_.size();
}
bool empty() {
std::lock_guard<std::mutex> guard(mtx_);
return tasks_.empty();
}
private:
std::queue<std::packaged_task<void()> > tasks_;
std::mutex mtx_;
}; };
class ThreadPool { template <class F, class... Args>
public: decltype(auto) addTask(F &&f, Args &&...args)
explicit ThreadPool(size_t nWorkers) {
: if (drain_)
tqit_(taskQueues_.begin()), stop_(false), drain_(false) { throw std::runtime_error("Unable to add task to draining pool");
resize(nWorkers); auto tq = std::make_shared<TaskQueue>();
}
~ThreadPool() { shutdown(); } auto fut = tq->addTask(f, args...);
void shutdown() { {
stop_ = true; std::lock_guard<std::mutex> guard(mtx_);
cv_.notify_all(); taskQueues_.push_back(tq);
for (std::thread &worker : workers_) { }
if (worker.joinable()) cv_.notify_one();
worker.join(); return fut;
} }
}
void drain() { void addTasks(std::shared_ptr<TaskQueue> tq)
drain_ = true; {
while (true) { if (drain_)
{ throw std::runtime_error("Unable to add task to draining pool");
std::lock_guard<std::mutex> guard(mtx_); std::lock_guard<std::mutex> guard(mtx_);
if (taskQueues_.empty()) break; taskQueues_.push_back(tq);
} cv_.notify_one();
std::this_thread::sleep_for(250ms); }
}
}
void restart() { private:
drain_ = false; // need to keep track of threads so we can join them
} std::vector<std::thread> workers_;
// the task queue
std::list<std::shared_ptr<TaskQueue>> taskQueues_;
std::list<std::shared_ptr<TaskQueue>>::iterator tqit_;
void resize(size_t nWorkers) { // synchronization
shutdown(); std::mutex mtx_;
workers_.clear(); std::condition_variable cv_;
stop_ = false; std::atomic<bool> stop_;
std::atomic<bool> drain_;
};
for (size_t i = 0; i < nWorkers; ++i) } // namespace daggy
workers_.emplace_back([&] {
while (true) {
std::packaged_task<void()> task;
{
std::unique_lock<std::mutex> lock(mtx_);
cv_.wait(lock, [&] { return stop_ || !taskQueues_.empty(); });
if (taskQueues_.empty()) {
if (stop_) return;
continue;
}
if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin();
task = std::move((*tqit_)->pop());
if ((*tqit_)->empty()) {
tqit_ = taskQueues_.erase(tqit_);
} else {
tqit_++;
}
}
task();
}
}
);
};
template<class F, class... Args>
decltype(auto) addTask(F &&f, Args &&... args) {
if (drain_) throw std::runtime_error("Unable to add task to draining pool");
auto tq = std::make_shared<TaskQueue>();
auto fut = tq->addTask(f, args...);
{
std::lock_guard<std::mutex> guard(mtx_);
taskQueues_.push_back(tq);
}
cv_.notify_one();
return fut;
}
void addTasks(std::shared_ptr<TaskQueue> tq) {
if (drain_) throw std::runtime_error("Unable to add task to draining pool");
std::lock_guard<std::mutex> guard(mtx_);
taskQueues_.push_back(tq);
cv_.notify_one();
}
private:
// need to keep track of threads so we can join them
std::vector<std::thread> workers_;
// the task queue
std::list<std::shared_ptr<TaskQueue>> taskQueues_;
std::list<std::shared_ptr<TaskQueue>>::iterator tqit_;
// synchronization
std::mutex mtx_;
std::condition_variable cv_;
std::atomic<bool> stop_;
std::atomic<bool> drain_;
};
}

52
daggy/include/daggy/Utilities.hpp
View File

@@ -1,41 +1,39 @@
#pragma once #pragma once
#include <vector>
#include <string>
#include <variant>
#include <unordered_map>
#include <rapidjson/document.h> #include <rapidjson/document.h>
#include "daggy/loggers/dag_run/DAGRunLogger.hpp" #include <string>
#include "daggy/executors/task/TaskExecutor.hpp" #include <unordered_map>
#include "Defines.hpp" #include <variant>
#include <vector>
#include "DAG.hpp" #include "DAG.hpp"
#include "Defines.hpp"
#include "daggy/executors/task/TaskExecutor.hpp"
#include "daggy/loggers/dag_run/DAGRunLogger.hpp"
namespace daggy { namespace daggy {
using TaskDAG = DAG<std::string, Task>; using TaskDAG = DAG<std::string, Task>;
std::string globalSub(std::string string, const std::string &pattern, const std::string &replacement); std::string globalSub(std::string string, const std::string &pattern,
const std::string &replacement);
std::vector<Command> interpolateValues(const std::vector<std::string> &raw, const ConfigValues &values); std::vector<Command> interpolateValues(const std::vector<std::string> &raw,
const ConfigValues &values);
TaskSet TaskSet expandTaskSet(const TaskSet &tasks,
expandTaskSet(const TaskSet &tasks, executors::task::TaskExecutor &executor,
executors::task::TaskExecutor &executor, const ConfigValues &interpolatedValues = {});
const ConfigValues &interpolatedValues = {});
TaskDAG buildDAGFromTasks(
TaskSet &tasks,
const std::vector<loggers::dag_run::TaskUpdateRecord> &updates = {});
TaskDAG void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks);
buildDAGFromTasks(TaskSet &tasks,
const std::vector<loggers::dag_run::TaskUpdateRecord> &updates = {});
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks); TaskDAG runDAG(DAGRunID runID, executors::task::TaskExecutor &executor,
loggers::dag_run::DAGRunLogger &logger, TaskDAG dag,
const ConfigValues job = {});
TaskDAG runDAG(DAGRunID runID, std::ostream &operator<<(std::ostream &os, const TimePoint &tp);
executors::task::TaskExecutor &executor, } // namespace daggy
loggers::dag_run::DAGRunLogger &logger,
TaskDAG dag,
const ConfigValues job = {});
std::ostream &operator<<(std::ostream &os, const TimePoint &tp);
}

40
daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp
View File

@@ -1,27 +1,33 @@
#pragma once #pragma once
#include "TaskExecutor.hpp"
#include <daggy/ThreadPool.hpp> #include <daggy/ThreadPool.hpp>
#include "TaskExecutor.hpp"
namespace daggy::executors::task { namespace daggy::executors::task {
class ForkingTaskExecutor : public TaskExecutor { class ForkingTaskExecutor : public TaskExecutor
public: {
using Command = std::vector<std::string>; public:
using Command = std::vector<std::string>;
ForkingTaskExecutor(size_t nThreads) ForkingTaskExecutor(size_t nThreads)
: tp_(nThreads) {} : tp_(nThreads)
{
}
// Validates the job to ensure that all required values are set and are of the right type, // Validates the job to ensure that all required values are set and are of
bool validateTaskParameters(const ConfigValues &job) override; // the right type,
bool validateTaskParameters(const ConfigValues &job) override;
std::vector<ConfigValues> std::vector<ConfigValues> expandTaskParameters(
expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; const ConfigValues &job, const ConfigValues &expansionValues) override;
// Runs the task // Runs the task
std::future<AttemptRecord> execute(const std::string &taskName, const Task &task) override; std::future<AttemptRecord> execute(const std::string &taskName,
const Task &task) override;
private: private:
ThreadPool tp_; ThreadPool tp_;
AttemptRecord runTask(const Task & task); AttemptRecord runTask(const Task &task);
}; };
} } // namespace daggy::executors::task

26
daggy/include/daggy/executors/task/NoopTaskExecutor.hpp
View File

@@ -3,18 +3,20 @@
#include "TaskExecutor.hpp" #include "TaskExecutor.hpp"
namespace daggy::executors::task { namespace daggy::executors::task {
class NoopTaskExecutor : public TaskExecutor { class NoopTaskExecutor : public TaskExecutor
public: {
using Command = std::vector<std::string>; public:
using Command = std::vector<std::string>;
// Validates the job to ensure that all required values are set and are of the right type, // Validates the job to ensure that all required values are set and are of
bool validateTaskParameters(const ConfigValues &job) override; // the right type,
bool validateTaskParameters(const ConfigValues &job) override;
std::vector<ConfigValues> std::vector<ConfigValues> expandTaskParameters(
expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; const ConfigValues &job, const ConfigValues &expansionValues) override;
// Runs the task
std::future<AttemptRecord> execute(const std::string &taskName, const Task &task) override;
};
}
// Runs the task
std::future<AttemptRecord> execute(const std::string &taskName,
const Task &task) override;
};
} // namespace daggy::executors::task

56
daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp
View File

@@ -3,35 +3,39 @@
#include "TaskExecutor.hpp" #include "TaskExecutor.hpp"
namespace daggy::executors::task { namespace daggy::executors::task {
class SlurmTaskExecutor : public TaskExecutor { class SlurmTaskExecutor : public TaskExecutor
public: {
using Command = std::vector<std::string>; public:
using Command = std::vector<std::string>;
SlurmTaskExecutor(); SlurmTaskExecutor();
~SlurmTaskExecutor(); ~SlurmTaskExecutor();
// Validates the job to ensure that all required values are set and are of the right type, // Validates the job to ensure that all required values are set and are of
bool validateTaskParameters(const ConfigValues &job) override; // the right type,
bool validateTaskParameters(const ConfigValues &job) override;
std::vector<ConfigValues> std::vector<ConfigValues> expandTaskParameters(
expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; const ConfigValues &job, const ConfigValues &expansionValues) override;
// Runs the task // Runs the task
std::future<AttemptRecord> execute(const std::string &taskName, const Task &task) override; std::future<AttemptRecord> execute(const std::string &taskName,
const Task &task) override;
private: private:
struct Job { struct Job
std::promise<AttemptRecord> prom; {
std::string stdoutFile; std::promise<AttemptRecord> prom;
std::string stderrFile; std::string stdoutFile;
}; std::string stderrFile;
std::mutex promiseGuard_;
std::unordered_map<size_t, Job> runningJobs_;
std::atomic<bool> running_;
// Monitors jobs and resolves promises
std::thread monitorWorker_;
void monitor();
}; };
}
std::mutex promiseGuard_;
std::unordered_map<size_t, Job> runningJobs_;
std::atomic<bool> running_;
// Monitors jobs and resolves promises
std::thread monitorWorker_;
void monitor();
};
} // namespace daggy::executors::task

30
daggy/include/daggy/executors/task/TaskExecutor.hpp
View File

@@ -1,31 +1,33 @@
#pragma once #pragma once
#include <chrono> #include <chrono>
#include <daggy/Defines.hpp>
#include <future> #include <future>
#include <string> #include <string>
#include <thread> #include <thread>
#include <vector> #include <vector>
#include <daggy/Defines.hpp>
/* /*
Executors run Tasks, returning a future with the results. Executors run Tasks, returning a future with the results.
If there are many retries, logs are returned for each attempt. If there are many retries, logs are returned for each attempt.
*/ */
namespace daggy::executors::task { namespace daggy::executors::task {
class TaskExecutor { class TaskExecutor
public: {
virtual ~TaskExecutor() = default; public:
virtual ~TaskExecutor() = default;
// Validates the job to ensure that all required values are set and are of the right type, // Validates the job to ensure that all required values are set and are of
virtual bool validateTaskParameters(const ConfigValues &job) = 0; // the right type,
virtual bool validateTaskParameters(const ConfigValues &job) = 0;
// Will use the expansion values to return the fully expanded tasks. // Will use the expansion values to return the fully expanded tasks.
virtual std::vector<ConfigValues> virtual std::vector<ConfigValues> expandTaskParameters(
expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) = 0; const ConfigValues &job, const ConfigValues &expansionValues) = 0;
// Blocking execution of a task // Blocking execution of a task
virtual std::future<AttemptRecord> execute(const std::string &taskName, const Task &task) = 0; virtual std::future<AttemptRecord> execute(const std::string &taskName,
}; const Task &task) = 0;
} };
} // namespace daggy::executors::task

42
daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp
View File

@@ -11,32 +11,32 @@
be supported. be supported.
*/ */
namespace daggy { namespace daggy { namespace loggers { namespace dag_run {
namespace loggers { class DAGRunLogger
namespace dag_run { {
class DAGRunLogger { public:
public: virtual ~DAGRunLogger() = default;
virtual ~DAGRunLogger() = default;
// Execution // Execution
virtual DAGRunID startDAGRun(std::string name, const TaskSet &tasks) = 0; virtual DAGRunID startDAGRun(std::string name, const TaskSet &tasks) = 0;
virtual void addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) = 0; virtual void addTask(DAGRunID dagRunID, const std::string taskName,
const Task &task) = 0;
virtual void updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) = 0; virtual void updateTask(DAGRunID dagRunID, const std::string taskName,
const Task &task) = 0;
virtual void updateDAGRunState(DAGRunID dagRunID, RunState state) = 0; virtual void updateDAGRunState(DAGRunID dagRunID, RunState state) = 0;
virtual void virtual void logTaskAttempt(DAGRunID dagRunID, const std::string &taskName,
logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, const AttemptRecord &attempt) = 0; const AttemptRecord &attempt) = 0;
virtual void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) = 0; virtual void updateTaskState(DAGRunID dagRunID, const std::string &taskName,
RunState state) = 0;
// Querying // Querying
virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) = 0; virtual std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) = 0;
virtual DAGRunRecord getDAGRun(DAGRunID dagRunID) = 0; virtual DAGRunRecord getDAGRun(DAGRunID dagRunID) = 0;
}; };
} }}} // namespace daggy::loggers::dag_run
}
}

62
daggy/include/daggy/loggers/dag_run/Defines.hpp
View File

@@ -2,40 +2,44 @@
#include <cstdint> #include <cstdint>
#include <string> #include <string>
#include <vector>
#include <unordered_set>
#include <unordered_map> #include <unordered_map>
#include <unordered_set>
#include <vector>
#include "../../Defines.hpp" #include "../../Defines.hpp"
namespace daggy::loggers::dag_run { namespace daggy::loggers::dag_run {
struct TaskUpdateRecord { struct TaskUpdateRecord
TimePoint time; {
std::string taskName; TimePoint time;
RunState newState; std::string taskName;
}; RunState newState;
};
struct DAGUpdateRecord { struct DAGUpdateRecord
TimePoint time; {
RunState newState; TimePoint time;
}; RunState newState;
};
// Pretty heavy weight, but // Pretty heavy weight, but
struct DAGRunRecord { struct DAGRunRecord
std::string name; {
TaskSet tasks; std::string name;
std::unordered_map<std::string, RunState> taskRunStates; TaskSet tasks;
std::unordered_map<std::string, std::vector<AttemptRecord>> taskAttempts; std::unordered_map<std::string, RunState> taskRunStates;
std::vector<TaskUpdateRecord> taskStateChanges; std::unordered_map<std::string, std::vector<AttemptRecord>> taskAttempts;
std::vector<DAGUpdateRecord> dagStateChanges; std::vector<TaskUpdateRecord> taskStateChanges;
}; std::vector<DAGUpdateRecord> dagStateChanges;
};
struct DAGRunSummary { struct DAGRunSummary
DAGRunID runID; {
std::string name; DAGRunID runID;
RunState runState; std::string name;
TimePoint startTime; RunState runState;
TimePoint lastUpdate; TimePoint startTime;
std::unordered_map<RunState, size_t> taskStateCounts; TimePoint lastUpdate;
}; std::unordered_map<RunState, size_t> taskStateCounts;
} };
} // namespace daggy::loggers::dag_run

91
daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp
View File

@@ -1,10 +1,11 @@
#pragma once #pragma once
#include <filesystem> #include <rapidjson/document.h>
#include <atomic> #include <atomic>
#include <filesystem>
#include <mutex> #include <mutex>
#include <rapidjson/document.h>
#include "DAGRunLogger.hpp" #include "DAGRunLogger.hpp"
#include "Defines.hpp" #include "Defines.hpp"
@@ -12,56 +13,58 @@ namespace fs = std::filesystem;
namespace rj = rapidjson; namespace rj = rapidjson;
namespace daggy::loggers::dag_run { namespace daggy::loggers::dag_run {
/* /*
* This logger should only be used for debug purposes. It's not really optimized for querying, and will * This logger should only be used for debug purposes. It's not really
* use a ton of inodes to track state. * optimized for querying, and will use a ton of inodes to track state.
* *
* On the plus side, it's trivial to look at without using the API. * On the plus side, it's trivial to look at without using the API.
* *
* Filesystem logger creates the following structure: * Filesystem logger creates the following structure:
* {root}/ * {root}/
* runs/ * runs/
* {runID}/ * {runID}/
* meta.json --- Contains the DAG name, task definitions * meta.json --- Contains the DAG name, task definitions
* states.csv --- DAG state changes * states.csv --- DAG state changes
* {taskName}/ * {taskName}/
* states.csv --- TASK state changes * states.csv --- TASK state changes
* {attempt}/ * {attempt}/
* metadata.json --- timestamps and rc * metadata.json --- timestamps and rc
* output.log * output.log
* error.log * error.log
* executor.log * executor.log
*/ */
class FileSystemLogger : public DAGRunLogger { class FileSystemLogger : public DAGRunLogger
public: {
FileSystemLogger(fs::path root); public:
FileSystemLogger(fs::path root);
// Execution // Execution
DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override;
void updateDAGRunState(DAGRunID dagRunID, RunState state) override; void updateDAGRunState(DAGRunID dagRunID, RunState state) override;
void void logTaskAttempt(DAGRunID, const std::string &taskName,
logTaskAttempt(DAGRunID, const std::string &taskName, const AttemptRecord &attempt) override; const AttemptRecord &attempt) override;
void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) override; void updateTaskState(DAGRunID dagRunID, const std::string &taskName,
RunState state) override;
// Querying // Querying
std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override; std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
DAGRunRecord getDAGRun(DAGRunID dagRunID) override; DAGRunRecord getDAGRun(DAGRunID dagRunID) override;
private: private:
fs::path root_; fs::path root_;
std::atomic<DAGRunID> nextRunID_; std::atomic<DAGRunID> nextRunID_;
std::mutex lock_; std::mutex lock_;
// std::unordered_map<fs::path, std::mutex> runLocks; // std::unordered_map<fs::path, std::mutex> runLocks;
inline const fs::path getCurrentPath() const; inline const fs::path getCurrentPath() const;
inline const fs::path getRunsRoot() const; inline const fs::path getRunsRoot() const;
inline const fs::path getRunRoot(DAGRunID runID) const; inline const fs::path getRunRoot(DAGRunID runID) const;
}; };
} } // namespace daggy::loggers::dag_run

63
daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp
View File

@@ -6,45 +6,46 @@
#include "DAGRunLogger.hpp" #include "DAGRunLogger.hpp"
#include "Defines.hpp" #include "Defines.hpp"
namespace daggy { namespace daggy { namespace loggers { namespace dag_run {
namespace loggers { /*
namespace dag_run { * This logger should only be used for debug purposes. It doesn't actually log
/* * anything, just prints stuff to stdout.
* This logger should only be used for debug purposes. It doesn't actually log anything, just prints stuff */
* to stdout. class OStreamLogger : public DAGRunLogger
*/ {
class OStreamLogger : public DAGRunLogger { public:
public: OStreamLogger(std::ostream &os);
OStreamLogger(std::ostream &os);
// Execution // Execution
DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override;
void addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) override; void addTask(DAGRunID dagRunID, const std::string taskName,
const Task &task) override;
void updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) override; void updateTask(DAGRunID dagRunID, const std::string taskName,
const Task &task) override;
void updateDAGRunState(DAGRunID dagRunID, RunState state) override; void updateDAGRunState(DAGRunID dagRunID, RunState state) override;
void void logTaskAttempt(DAGRunID, const std::string &taskName,
logTaskAttempt(DAGRunID, const std::string &taskName, const AttemptRecord &attempt) override; const AttemptRecord &attempt) override;
void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) override; void updateTaskState(DAGRunID dagRunID, const std::string &taskName,
RunState state) override;
// Querying // Querying
std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override; std::vector<DAGRunSummary> getDAGs(uint32_t stateMask) override;
DAGRunRecord getDAGRun(DAGRunID dagRunID) override; DAGRunRecord getDAGRun(DAGRunID dagRunID) override;
private: private:
std::mutex guard_; std::mutex guard_;
std::ostream &os_; std::ostream &os_;
std::vector<DAGRunRecord> dagRuns_; std::vector<DAGRunRecord> dagRuns_;
void _updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state); void _updateTaskState(DAGRunID dagRunID, const std::string &taskName,
RunState state);
void _updateDAGRunState(DAGRunID dagRunID, RunState state); void _updateDAGRunState(DAGRunID dagRunID, RunState state);
}; };
} }}} // namespace daggy::loggers::dag_run
}
}

466
daggy/src/Serialization.cpp
View File

@@ -1,252 +1,292 @@
#include <sstream>
#include <iomanip>
#include <rapidjson/error/en.h> #include <rapidjson/error/en.h>
#include <daggy/Serialization.hpp> #include <daggy/Serialization.hpp>
#include <daggy/Utilities.hpp> #include <daggy/Utilities.hpp>
#include <iomanip>
#include <sstream>
namespace daggy { namespace daggy {
void checkRJParse(const rj::ParseResult &result, const std::string &prefix) { void checkRJParse(const rj::ParseResult &result, const std::string &prefix)
if (!result) { {
std::stringstream ss; if (!result) {
ss << (prefix.empty() ? "" : prefix + ':') std::stringstream ss;
<< "Error parsing JSON: " << rj::GetParseError_En(result.Code()) ss << (prefix.empty() ? "" : prefix + ':')
<< " at byte offset " << result.Offset(); << "Error parsing JSON: " << rj::GetParseError_En(result.Code())
throw std::runtime_error(ss.str()); << " at byte offset " << result.Offset();
throw std::runtime_error(ss.str());
}
}
ConfigValues configFromJSON(const std::string &jsonSpec)
{
rj::Document doc;
checkRJParse(doc.Parse(jsonSpec.c_str()), "Parsing config");
return configFromJSON(doc);
}
ConfigValues configFromJSON(const rj::Value &spec)
{
std::unordered_map<std::string, ConfigValue> parameters;
if (!spec.IsObject()) {
throw std::runtime_error("Parameters in spec is not a JSON dictionary");
}
for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) {
if (!it->name.IsString()) {
throw std::runtime_error("All keys must be strings.");
}
std::string name = it->name.GetString();
if (it->value.IsArray()) {
std::vector<std::string> values;
for (size_t i = 0; i < it->value.Size(); ++i) {
if (!it->value[i].IsString()) {
throw std::runtime_error(
"Attribute for " + std::string{it->name.GetString()} +
" item " + std::to_string(i) + " is not a string.");
}
values.emplace_back(it->value[i].GetString());
} }
parameters[name] = values;
}
else if (it->value.IsString()) {
parameters[name] = it->value.GetString();
}
else {
throw std::runtime_error("Attribute for " +
std::string{it->name.GetString()} +
" is not a string or an array.");
}
}
return parameters;
}
std::string configToJSON(const ConfigValues &config)
{
std::stringstream ss;
ss << '{';
bool first = true;
for (const auto &[k, v] : config) {
if (first) {
first = false;
}
else {
ss << ", ";
}
ss << std::quoted(k) << ": ";
if (std::holds_alternative<std::string>(v)) {
ss << std::quoted(std::get<std::string>(v));
}
else {
ss << '[';
const auto &vals = std::get<std::vector<std::string>>(v);
bool firstVal = true;
for (const auto &val : vals) {
if (firstVal) {
firstVal = false;
}
else {
ss << ", ";
}
ss << std::quoted(val);
}
ss << ']';
}
}
ss << '}';
return ss.str();
}
Task taskFromJSON(const std::string &name, const rj::Value &spec,
const ConfigValues &jobDefaults)
{
Task task{.definedName = name,
.isGenerator = false,
.maxRetries = 0,
.retryIntervalSeconds = 0,
.job = jobDefaults};
if (!spec.IsObject()) {
throw std::runtime_error("Tasks is not an object");
} }
ConfigValues configFromJSON(const std::string &jsonSpec) { // Grab the standard fields with defaults;
rj::Document doc; if (spec.HasMember("isGenerator")) {
checkRJParse(doc.Parse(jsonSpec.c_str()), "Parsing config"); task.isGenerator = spec["isGenerator"].GetBool();
return configFromJSON(doc);
} }
ConfigValues configFromJSON(const rj::Value &spec) { if (spec.HasMember("maxRetries")) {
std::unordered_map<std::string, ConfigValue> parameters; task.maxRetries = spec["maxRetries"].GetInt();
if (!spec.IsObject()) { throw std::runtime_error("Parameters in spec is not a JSON dictionary"); }
for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) {
if (!it->name.IsString()) {
throw std::runtime_error("All keys must be strings.");
}
std::string name = it->name.GetString();
if (it->value.IsArray()) {
std::vector<std::string> values;
for (size_t i = 0; i < it->value.Size(); ++i) {
if (!it->value[i].IsString()) {
throw std::runtime_error(
"Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) +
" is not a string.");
}
values.emplace_back(it->value[i].GetString());
}
parameters[name] = values;
} else if (it->value.IsString()) {
parameters[name] = it->value.GetString();
} else {
throw std::runtime_error(
"Attribute for " + std::string{it->name.GetString()} + " is not a string or an array.");
}
}
return parameters;
} }
std::string configToJSON(const ConfigValues &config) { if (spec.HasMember("retryIntervalSeconds")) {
std::stringstream ss; task.retryIntervalSeconds = spec["retryIntervalSeconds"].GetInt();
ss << '{';
bool first = true;
for (const auto &[k, v]: config) {
if (first) { first = false; } else { ss << ", "; }
ss << std::quoted(k) << ": ";
if (std::holds_alternative<std::string>(v)) {
ss << std::quoted(std::get<std::string>(v));
} else {
ss << '[';
const auto &vals = std::get<std::vector<std::string>>(v);
bool firstVal = true;
for (const auto &val: vals) {
if (firstVal) { firstVal = false; } else { ss << ", "; }
ss << std::quoted(val);
}
ss << ']';
}
}
ss << '}';
return ss.str();
} }
Task // Children / parents
taskFromJSON(const std::string &name, const rj::Value &spec, const ConfigValues &jobDefaults) { if (spec.HasMember("children")) {
Task task{ const auto &specChildren = spec["children"].GetArray();
.definedName = name, for (size_t c = 0; c < specChildren.Size(); ++c) {
.isGenerator = false, task.children.insert(specChildren[c].GetString());
.maxRetries = 0, }
.retryIntervalSeconds = 0,
.job = jobDefaults
};
if (!spec.IsObject()) { throw std::runtime_error("Tasks is not an object"); }
// Grab the standard fields with defaults;
if (spec.HasMember("isGenerator")) {
task.isGenerator = spec["isGenerator"].GetBool();
}
if (spec.HasMember("maxRetries")) {
task.maxRetries = spec["maxRetries"].GetInt();
}
if (spec.HasMember("retryIntervalSeconds")) {
task.retryIntervalSeconds = spec["retryIntervalSeconds"].GetInt();
}
// Children / parents
if (spec.HasMember("children")) {
const auto &specChildren = spec["children"].GetArray();
for (size_t c = 0; c < specChildren.Size(); ++c) {
task.children.insert(specChildren[c].GetString());
}
}
if (spec.HasMember("parents")) {
const auto &specParents = spec["parents"].GetArray();
for (size_t c = 0; c < specParents.Size(); ++c) {
task.parents.insert(specParents[c].GetString());
}
}
if (spec.HasMember("job")) {
const auto &params = spec["job"];
if (!params.IsObject()) throw std::runtime_error("job is not a dictionary.");
for (auto it = params.MemberBegin(); it != params.MemberEnd(); ++it) {
if (!it->name.IsString()) throw std::runtime_error("job key must be a string.");
if (it->value.IsArray()) {
std::vector<std::string> values;
for (size_t i = 0; i < it->value.Size(); ++i) {
values.emplace_back(it->value[i].GetString());
}
task.job.insert_or_assign(it->name.GetString(), values);
} else {
task.job.insert_or_assign(it->name.GetString(), it->value.GetString());
}
}
}
return task;
} }
TaskSet tasksFromJSON(const std::string &jsonSpec, const ConfigValues &jobDefaults) { if (spec.HasMember("parents")) {
rj::Document doc; const auto &specParents = spec["parents"].GetArray();
checkRJParse(doc.Parse(jsonSpec.c_str())); for (size_t c = 0; c < specParents.Size(); ++c) {
return tasksFromJSON(doc, jobDefaults); task.parents.insert(specParents[c].GetString());
}
} }
TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults) { if (spec.HasMember("job")) {
TaskSet tasks; const auto &params = spec["job"];
if (!spec.IsObject()) { throw std::runtime_error("Tasks is not an object"); } if (!params.IsObject())
throw std::runtime_error("job is not a dictionary.");
// Tasks for (auto it = params.MemberBegin(); it != params.MemberEnd(); ++it) {
for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) { if (!it->name.IsString())
if (!it->name.IsString()) throw std::runtime_error("Task names must be a string."); throw std::runtime_error("job key must be a string.");
if (!it->value.IsObject()) throw std::runtime_error("Task definitions must be an object."); if (it->value.IsArray()) {
const auto &taskName = it->name.GetString(); std::vector<std::string> values;
tasks.emplace(taskName, taskFromJSON(taskName, it->value, jobDefaults)); for (size_t i = 0; i < it->value.Size(); ++i) {
values.emplace_back(it->value[i].GetString());
}
task.job.insert_or_assign(it->name.GetString(), values);
} }
else {
// Normalize tasks so all the children are populated task.job.insert_or_assign(it->name.GetString(),
for (auto &[k, v] : tasks) { it->value.GetString());
for (const auto & p : v.parents) {
tasks[p].children.insert(k);
}
v.parents.clear();
} }
}
return tasks;
} }
// I really want to do this with rapidjson, but damn they make it ugly and difficult. return task;
// So we'll shortcut and generate the JSON directly. }
std::string taskToJSON(const Task &task) {
std::stringstream ss;
bool first = false;
ss << "{" TaskSet tasksFromJSON(const std::string &jsonSpec,
<< R"("maxRetries": )" << task.maxRetries << ',' const ConfigValues &jobDefaults)
<< R"("retryIntervalSeconds": )" << task.retryIntervalSeconds << ','; {
rj::Document doc;
checkRJParse(doc.Parse(jsonSpec.c_str()));
return tasksFromJSON(doc, jobDefaults);
}
ss << R"("job": )" << configToJSON(task.job) << ','; TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults)
{
ss << R"("children": [)"; TaskSet tasks;
first = true; if (!spec.IsObject()) {
for (const auto &child: task.children) { throw std::runtime_error("Tasks is not an object");
if (!first) ss << ',';
ss << std::quoted(child);
first = false;
}
ss << "],";
ss << R"("parents": [)";
first = true;
for (const auto &parent: task.parents) {
if (!first) ss << ',';
ss << std::quoted(parent);
first = false;
}
ss << "],";
ss << R"("isGenerator": )" << (task.isGenerator ? "true" : "false");
ss << '}';
return ss.str();
} }
std::string tasksToJSON(const TaskSet &tasks) { // Tasks
std::stringstream ss; for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) {
if (!it->name.IsString())
ss << "{"; throw std::runtime_error("Task names must be a string.");
if (!it->value.IsObject())
bool first = true; throw std::runtime_error("Task definitions must be an object.");
for (const auto &[name, task]: tasks) { const auto &taskName = it->name.GetString();
if (!first) ss << ','; tasks.emplace(taskName, taskFromJSON(taskName, it->value, jobDefaults));
ss << std::quoted(name) << ": " << taskToJSON(task);
first = false;
}
ss << "}";
return ss.str();
} }
std::ostream &operator<<(std::ostream &os, const Task &task) { // Normalize tasks so all the children are populated
os << taskToJSON(task); for (auto &[k, v] : tasks) {
return os; for (const auto &p : v.parents) {
tasks[p].children.insert(k);
}
v.parents.clear();
} }
std::string attemptRecordToJSON(const AttemptRecord &record) { return tasks;
std::stringstream ss; }
ss << "{" // I really want to do this with rapidjson, but damn they make it ugly and
<< R"("startTime": )" << std::quoted(timePointToString(record.startTime)) << ',' // difficult. So we'll shortcut and generate the JSON directly.
<< R"("stopTime": )" << std::quoted(timePointToString(record.stopTime)) << ',' std::string taskToJSON(const Task &task)
<< R"("rc": )" << std::to_string(record.rc) << ',' {
<< R"("executorLog": )" << std::quoted(record.executorLog) << ',' std::stringstream ss;
<< R"("outputLog": )" << std::quoted(record.outputLog) << ',' bool first = false;
<< R"("errorLog": )" << std::quoted(record.errorLog)
<< '}';
return ss.str(); ss << "{"
<< R"("maxRetries": )" << task.maxRetries << ','
<< R"("retryIntervalSeconds": )" << task.retryIntervalSeconds << ',';
ss << R"("job": )" << configToJSON(task.job) << ',';
ss << R"("children": [)";
first = true;
for (const auto &child : task.children) {
if (!first)
ss << ',';
ss << std::quoted(child);
first = false;
} }
ss << "],";
std::string timePointToString(const TimePoint &tp) { ss << R"("parents": [)";
std::stringstream ss; first = true;
ss << tp; for (const auto &parent : task.parents) {
return ss.str(); if (!first)
ss << ',';
ss << std::quoted(parent);
first = false;
} }
ss << "],";
TimePoint stringToTimePoint(const std::string &timeString) { ss << R"("isGenerator": )" << (task.isGenerator ? "true" : "false");
std::tm dt;
std::stringstream ss{timeString}; ss << '}';
ss >> std::get_time(&dt, "%Y-%m-%d %H:%M:%S %Z"); return ss.str();
return Clock::from_time_t(mktime(&dt)); }
std::string tasksToJSON(const TaskSet &tasks)
{
std::stringstream ss;
ss << "{";
bool first = true;
for (const auto &[name, task] : tasks) {
if (!first)
ss << ',';
ss << std::quoted(name) << ": " << taskToJSON(task);
first = false;
} }
ss << "}";
} return ss.str();
}
std::ostream &operator<<(std::ostream &os, const Task &task)
{
os << taskToJSON(task);
return os;
}
std::string attemptRecordToJSON(const AttemptRecord &record)
{
std::stringstream ss;
ss << "{"
<< R"("startTime": )" << std::quoted(timePointToString(record.startTime))
<< ',' << R"("stopTime": )"
<< std::quoted(timePointToString(record.stopTime)) << ',' << R"("rc": )"
<< std::to_string(record.rc) << ',' << R"("executorLog": )"
<< std::quoted(record.executorLog) << ',' << R"("outputLog": )"
<< std::quoted(record.outputLog) << ',' << R"("errorLog": )"
<< std::quoted(record.errorLog) << '}';
return ss.str();
}
std::string timePointToString(const TimePoint &tp)
{
std::stringstream ss;
ss << tp;
return ss.str();
}
TimePoint stringToTimePoint(const std::string &timeString)
{
std::tm dt;
std::stringstream ss{timeString};
ss >> std::get_time(&dt, "%Y-%m-%d %H:%M:%S %Z");
return Clock::from_time_t(mktime(&dt));
}
} // namespace daggy

500
daggy/src/Server.cpp
View File

@@ -1,260 +1,308 @@
#include <iomanip>
#include <enum.h> #include <enum.h>
#include <daggy/Server.hpp>
#include <daggy/Serialization.hpp> #include <daggy/Serialization.hpp>
#include <daggy/Server.hpp>
#include <daggy/Utilities.hpp> #include <daggy/Utilities.hpp>
#include <iomanip>
#define REQ_ERROR(code, msg) response.send(Pistache::Http::Code::code, msg); return; #define REQ_ERROR(code, msg) \
response.send(Pistache::Http::Code::code, msg); \
return;
namespace rj = rapidjson; namespace rj = rapidjson;
using namespace Pistache; using namespace Pistache;
namespace daggy { namespace daggy {
void Server::init(int threads) { void Server::init(int threads)
auto opts = Http::Endpoint::options() {
.threads(threads) auto opts = Http::Endpoint::options()
.flags(Pistache::Tcp::Options::ReuseAddr | Pistache::Tcp::Options::ReusePort) .threads(threads)
.maxRequestSize(4294967296) .flags(Pistache::Tcp::Options::ReuseAddr |
.maxResponseSize(4294967296); Pistache::Tcp::Options::ReusePort)
endpoint_.init(opts); .maxRequestSize(4294967296)
createDescription(); .maxResponseSize(4294967296);
endpoint_.init(opts);
createDescription();
}
void Server::start()
{
router_.initFromDescription(desc_);
endpoint_.setHandler(router_.handler());
endpoint_.serveThreaded();
}
void Server::shutdown()
{
endpoint_.shutdown();
}
uint16_t Server::getPort() const
{
return endpoint_.getPort();
}
void Server::createDescription()
{
desc_.info().license("MIT", "https://opensource.org/licenses/MIT");
auto backendErrorResponse = desc_.response(
Http::Code::Internal_Server_Error, "An error occured with the backend");
desc_.schemes(Rest::Scheme::Http)
.basePath("/v1")
.produces(MIME(Application, Json))
.consumes(MIME(Application, Json));
desc_.route(desc_.get("/ready"))
.bind(&Server::handleReady, this)
.response(Http::Code::Ok, "Response to the /ready call")
.hide();
auto versionPath = desc_.path("/v1");
auto dagPath = versionPath.path("/dagrun");
// Run a DAG
dagPath.route(desc_.post("/"))
.bind(&Server::handleRunDAG, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "Run a DAG");
// List detailed DAG run
dagPath.route(desc_.get("/:runID"))
.bind(&Server::handleGetDAGRun, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "Details of a specific DAG run");
// List all DAG runs
dagPath.route(desc_.get("/"))
.bind(&Server::handleGetDAGRuns, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "The list of all known DAG Runs");
}
/*
* {
* "name": "DAG Run Name"
* "job": {...}
* "tasks": {...}
*/
void Server::handleRunDAG(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response)
{
if (!handleAuth(request, response))
return;
rj::Document doc;
try {
doc.Parse(request.body().c_str());
}
catch (std::exception &e) {
REQ_ERROR(Bad_Request, std::string{"Invalid JSON payload: "} + e.what());
} }
void Server::start() { if (!doc.IsObject()) {
router_.initFromDescription(desc_); REQ_ERROR(Bad_Request, "Payload is not a dictionary.");
}
endpoint_.setHandler(router_.handler()); if (!doc.HasMember("name")) {
endpoint_.serveThreaded(); REQ_ERROR(Bad_Request, "DAG Run is missing a name.");
}
if (!doc.HasMember("tasks")) {
REQ_ERROR(Bad_Request, "DAG Run has no tasks.");
} }
void Server::shutdown() { std::string runName = doc["name"].GetString();
endpoint_.shutdown();
// Get parameters if there are any
ConfigValues parameters;
if (doc.HasMember("parameters")) {
try {
auto parsedParams = configFromJSON(doc["parameters"].GetObject());
parameters.swap(parsedParams);
}
catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
}
} }
uint16_t Server::getPort() const { // Job Defaults
return endpoint_.getPort(); ConfigValues jobDefaults;
if (doc.HasMember("jobDefaults")) {
try {
auto parsedJobDefaults = configFromJSON(doc["jobDefaults"].GetObject());
jobDefaults.swap(parsedJobDefaults);
}
catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
}
} }
void Server::createDescription() { // Get the tasks
desc_ TaskSet tasks;
.info() try {
.license("MIT", "https://opensource.org/licenses/MIT"); auto taskTemplates = tasksFromJSON(doc["tasks"], jobDefaults);
auto expandedTasks = expandTaskSet(taskTemplates, executor_, parameters);
tasks.swap(expandedTasks);
auto backendErrorResponse = desc_.response(Http::Code::Internal_Server_Error, }
"An error occured with the backend"); catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
desc_
.schemes(Rest::Scheme::Http)
.basePath("/v1")
.produces(MIME(Application, Json))
.consumes(MIME(Application, Json));
desc_
.route(desc_.get("/ready"))
.bind(&Server::handleReady, this)
.response(Http::Code::Ok, "Response to the /ready call")
.hide();
auto versionPath = desc_.path("/v1");
auto dagPath = versionPath.path("/dagrun");
// Run a DAG
dagPath
.route(desc_.post("/"))
.bind(&Server::handleRunDAG, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "Run a DAG");
// List detailed DAG run
dagPath
.route(desc_.get("/:runID"))
.bind(&Server::handleGetDAGRun, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "Details of a specific DAG run");
// List all DAG runs
dagPath
.route(desc_.get("/"))
.bind(&Server::handleGetDAGRuns, this)
.produces(MIME(Application, Json), MIME(Application, Xml))
.response(Http::Code::Ok, "The list of all known DAG Runs");
} }
/* // Get a run ID
* { auto runID = logger_.startDAGRun(runName, tasks);
* "name": "DAG Run Name" auto dag = buildDAGFromTasks(tasks);
* "job": {...}
* "tasks": {...}
*/
void Server::handleRunDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) {
if (!handleAuth(request, response)) return;
rj::Document doc; runnerPool_.addTask([this, parameters, runID, dag]() {
try { runDAG(runID, this->executor_, this->logger_, dag, parameters);
doc.Parse(request.body().c_str()); });
} catch (std::exception &e) {
REQ_ERROR(Bad_Request, std::string{"Invalid JSON payload: "} + e.what()); response.send(Pistache::Http::Code::Ok,
R"({"runID": )" + std::to_string(runID) + "}");
}
void Server::handleGetDAGRuns(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response)
{
if (!handleAuth(request, response))
return;
auto dagRuns = logger_.getDAGs(0);
std::stringstream ss;
ss << '[';
bool first = true;
for (const auto &run : dagRuns) {
if (first) {
first = false;
}
else {
ss << ", ";
}
ss << " {"
<< R"("runID": )" << run.runID << ',' << R"("name": )"
<< std::quoted(run.name) << ","
<< R"("startTime": )" << std::quoted(timePointToString(run.startTime))
<< ',' << R"("lastUpdate": )"
<< std::quoted(timePointToString(run.lastUpdate)) << ','
<< R"("taskCounts": {)";
bool firstState = true;
for (const auto &[state, count] : run.taskStateCounts) {
if (firstState) {
firstState = false;
} }
else {
if (!doc.IsObject()) { REQ_ERROR(Bad_Request, "Payload is not a dictionary."); } ss << ", ";
if (!doc.HasMember("name")) { REQ_ERROR(Bad_Request, "DAG Run is missing a name."); }
if (!doc.HasMember("tasks")) { REQ_ERROR(Bad_Request, "DAG Run has no tasks."); }
std::string runName = doc["name"].GetString();
// Get parameters if there are any
ConfigValues parameters;
if (doc.HasMember("parameters")) {
try {
auto parsedParams = configFromJSON(doc["parameters"].GetObject());
parameters.swap(parsedParams);
} catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
}
} }
ss << std::quoted(state._to_string()) << ':' << count;
// Job Defaults }
ConfigValues jobDefaults; ss << '}' // end of taskCounts
if (doc.HasMember("jobDefaults")) { << '}'; // end of item
try {
auto parsedJobDefaults = configFromJSON(doc["jobDefaults"].GetObject());
jobDefaults.swap(parsedJobDefaults);
} catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
}
}
// Get the tasks
TaskSet tasks;
try {
auto taskTemplates = tasksFromJSON(doc["tasks"], jobDefaults);
auto expandedTasks = expandTaskSet(taskTemplates, executor_, parameters);
tasks.swap(expandedTasks);
} catch (std::exception &e) {
REQ_ERROR(Bad_Request, e.what());
}
// Get a run ID
auto runID = logger_.startDAGRun(runName, tasks);
auto dag = buildDAGFromTasks(tasks);
runnerPool_.addTask(
[this, parameters, runID, dag]() { runDAG(runID, this->executor_, this->logger_, dag, parameters); });
response.send(Pistache::Http::Code::Ok, R"({"runID": )" + std::to_string(runID) + "}");
} }
void Server::handleGetDAGRuns(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { ss << ']';
if (!handleAuth(request, response)) return; response.send(Pistache::Http::Code::Ok, ss.str());
auto dagRuns = logger_.getDAGs(0); }
std::stringstream ss;
ss << '[';
bool first = true; void Server::handleGetDAGRun(const Pistache::Rest::Request &request,
for (const auto &run: dagRuns) { Pistache::Http::ResponseWriter response)
if (first) { {
first = false; if (!handleAuth(request, response))
} else { return;
ss << ", "; if (!request.hasParam(":runID")) {
} REQ_ERROR(Not_Found, "No runID provided in URL");
ss << " {"
<< R"("runID": )" << run.runID << ','
<< R"("name": )" << std::quoted(run.name) << ","
<< R"("startTime": )" << std::quoted(timePointToString(run.startTime)) << ','
<< R"("lastUpdate": )" << std::quoted(timePointToString(run.lastUpdate)) << ','
<< R"("taskCounts": {)";
bool firstState = true;
for (const auto &[state, count]: run.taskStateCounts) {
if (firstState) {
firstState = false;
} else {
ss << ", ";
}
ss << std::quoted(state._to_string()) << ':' << count;
}
ss << '}' // end of taskCounts
<< '}'; // end of item
}
ss << ']';
response.send(Pistache::Http::Code::Ok, ss.str());
} }
DAGRunID runID = request.param(":runID").as<size_t>();
auto run = logger_.getDAGRun(runID);
void Server::handleGetDAGRun(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { bool first = true;
if (!handleAuth(request, response)) return; std::stringstream ss;
if (!request.hasParam(":runID")) { REQ_ERROR(Not_Found, "No runID provided in URL"); } ss << "{"
DAGRunID runID = request.param(":runID").as<size_t>(); << R"("runID": )" << runID << ',' << R"("name": )"
auto run = logger_.getDAGRun(runID); << std::quoted(run.name) << ',' << R"("tasks": )"
<< tasksToJSON(run.tasks) << ',';
bool first = true; // task run states
std::stringstream ss; ss << R"("taskStates": { )";
ss << "{" first = true;
<< R"("runID": )" << runID << ',' for (const auto &[name, state] : run.taskRunStates) {
<< R"("name": )" << std::quoted(run.name) << ',' if (first) {
<< R"("tasks": )" << tasksToJSON(run.tasks) << ','; first = false;
}
// task run states else {
ss << R"("taskStates": { )"; ss << ',';
first = true; }
for (const auto &[name, state]: run.taskRunStates) { ss << std::quoted(name) << ": " << std::quoted(state._to_string());
if (first) { first = false; } else { ss << ','; }
ss << std::quoted(name) << ": " << std::quoted(state._to_string());
}
ss << "},";
// Attempt records
first = true;
ss << R"("taskAttempts": { )";
for (const auto &[taskName, attempts]: run.taskAttempts) {
if (first) { first = false; } else { ss << ','; }
ss << std::quoted(taskName) << ": [";
bool firstAttempt = true;
for (const auto &attempt: attempts) {
if (firstAttempt) { firstAttempt = false; } else { ss << ','; }
ss << '{'
<< R"("startTime":)" << std::quoted(timePointToString(attempt.startTime)) << ','
<< R"("stopTime":)" << std::quoted(timePointToString(attempt.stopTime)) << ','
<< R"("rc":)" << attempt.rc << ','
<< R"("outputLog":)" << std::quoted(attempt.outputLog) << ','
<< R"("errorLog":)" << std::quoted(attempt.errorLog) << ','
<< R"("executorLog":)" << std::quoted(attempt.executorLog)
<< '}';
}
ss << ']';
}
ss << "},";
// DAG state changes
first = true;
ss << R"("dagStateChanges": [ )";
for (const auto &change: run.dagStateChanges) {
if (first) { first = false; } else { ss << ','; }
ss << '{'
<< R"("newState": )" << std::quoted(change.newState._to_string()) << ','
<< R"("time": )" << std::quoted(timePointToString(change.time))
<< '}';
}
ss << "]";
ss << '}';
response.send(Pistache::Http::Code::Ok, ss.str());
} }
ss << "},";
void Server::handleReady(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { // Attempt records
response.send(Pistache::Http::Code::Ok, "Ya like DAGs?"); first = true;
ss << R"("taskAttempts": { )";
for (const auto &[taskName, attempts] : run.taskAttempts) {
if (first) {
first = false;
}
else {
ss << ',';
}
ss << std::quoted(taskName) << ": [";
bool firstAttempt = true;
for (const auto &attempt : attempts) {
if (firstAttempt) {
firstAttempt = false;
}
else {
ss << ',';
}
ss << '{' << R"("startTime":)"
<< std::quoted(timePointToString(attempt.startTime)) << ','
<< R"("stopTime":)"
<< std::quoted(timePointToString(attempt.stopTime)) << ','
<< R"("rc":)" << attempt.rc << ',' << R"("outputLog":)"
<< std::quoted(attempt.outputLog) << ',' << R"("errorLog":)"
<< std::quoted(attempt.errorLog) << ',' << R"("executorLog":)"
<< std::quoted(attempt.executorLog) << '}';
}
ss << ']';
} }
ss << "},";
/* // DAG state changes
* handleAuth will check any auth methods and handle any responses in the case of failed auth. If it returns first = true;
* false, callers should cease handling the response ss << R"("dagStateChanges": [ )";
*/ for (const auto &change : run.dagStateChanges) {
bool Server::handleAuth(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response) { if (first) {
(void) response; first = false;
return true; }
else {
ss << ',';
}
ss << '{' << R"("newState": )"
<< std::quoted(change.newState._to_string()) << ',' << R"("time": )"
<< std::quoted(timePointToString(change.time)) << '}';
} }
} ss << "]";
ss << '}';
response.send(Pistache::Http::Code::Ok, ss.str());
}
void Server::handleReady(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter response)
{
response.send(Pistache::Http::Code::Ok, "Ya like DAGs?");
}
/*
* handleAuth will check any auth methods and handle any responses in the case
* of failed auth. If it returns false, callers should cease handling the
* response
*/
bool Server::handleAuth(const Pistache::Rest::Request &request,
Pistache::Http::ResponseWriter &response)
{
(void)response;
return true;
}
} // namespace daggy

411
daggy/src/Utilities.cpp
View File

@@ -1,219 +1,234 @@
#include <daggy/Serialization.hpp>
#include <daggy/Utilities.hpp>
#include <future> #include <future>
#include <iomanip> #include <iomanip>
#include <daggy/Utilities.hpp>
#include <daggy/Serialization.hpp>
using namespace std::chrono_literals; using namespace std::chrono_literals;
namespace daggy { namespace daggy {
std::string globalSub(std::string string, const std::string &pattern, const std::string &replacement) { std::string globalSub(std::string string, const std::string &pattern,
size_t pos = string.find(pattern); const std::string &replacement)
while (pos != std::string::npos) { {
string.replace(pos, pattern.size(), replacement); size_t pos = string.find(pattern);
pos = string.find(pattern); while (pos != std::string::npos) {
string.replace(pos, pattern.size(), replacement);
pos = string.find(pattern);
}
return string;
}
std::vector<std::vector<std::string>> interpolateValues(
const std::vector<std::string> &raw, const ConfigValues &values)
{
std::vector<std::vector<std::string>> cooked{{}};
for (const auto &part : raw) {
std::vector<std::string> expandedPart{part};
// Find all values of parameters, and expand them
for (const auto &[paramRaw, paramValue] : values) {
std::string param = "{{" + paramRaw + "}}";
auto pos = part.find(param);
if (pos == std::string::npos)
continue;
std::vector<std::string> newExpandedPart;
if (std::holds_alternative<std::string>(paramValue)) {
for (auto &cmd : expandedPart) {
newExpandedPart.push_back(
globalSub(cmd, param, std::get<std::string>(paramValue)));
}
} }
return string; else {
for (const auto &val :
std::get<std::vector<std::string>>(paramValue)) {
for (auto cmd : expandedPart) {
newExpandedPart.push_back(globalSub(cmd, param, val));
}
}
}
expandedPart.swap(newExpandedPart);
}
std::vector<std::vector<std::string>> newCommands;
for (const auto &newPart : expandedPart) {
for (auto cmd : cooked) {
cmd.push_back(newPart);
newCommands.emplace_back(cmd);
}
}
cooked.swap(newCommands);
}
return cooked;
}
TaskSet expandTaskSet(const TaskSet &tasks,
executors::task::TaskExecutor &executor,
const ConfigValues &interpolatedValues)
{
// Expand the tasks first
TaskSet newTaskSet;
for (const auto &[baseName, task] : tasks) {
executor.validateTaskParameters(task.job);
const auto newJobs =
executor.expandTaskParameters(task.job, interpolatedValues);
size_t i = 0;
for (const auto &newJob : newJobs) {
Task newTask{task};
newTask.job = newJob;
newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask);
++i;
}
}
return newTaskSet;
}
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks)
{
// Add the missing vertices
for (const auto &[name, task] : tasks) {
dag.addVertex(name, task);
} }
std::vector<std::vector<std::string>> // Add edges
interpolateValues(const std::vector<std::string> &raw, const ConfigValues &values) { for (const auto &[name, task] : tasks) {
std::vector<std::vector<std::string>> cooked{{}}; dag.addEdgeIf(name, [&](const auto &v) {
return task.children.count(v.data.definedName) > 0;
});
}
for (const auto &part: raw) { if (!dag.isValid()) {
std::vector<std::string> expandedPart{part}; throw std::runtime_error("DAG contains a cycle");
}
}
// Find all values of parameters, and expand them TaskDAG buildDAGFromTasks(
for (const auto &[paramRaw, paramValue]: values) { TaskSet &tasks,
std::string param = "{{" + paramRaw + "}}"; const std::vector<loggers::dag_run::TaskUpdateRecord> &updates)
auto pos = part.find(param); {
if (pos == std::string::npos) continue; TaskDAG dag;
std::vector<std::string> newExpandedPart; updateDAGFromTasks(dag, tasks);
if (std::holds_alternative<std::string>(paramValue)) { // Replay any updates
for (auto &cmd: expandedPart) { for (const auto &update : updates) {
newExpandedPart.push_back(globalSub(cmd, param, std::get<std::string>(paramValue))); switch (update.newState) {
} case RunState::RUNNING:
} else { case RunState::RETRY:
for (const auto &val: std::get<std::vector<std::string>>(paramValue)) { case RunState::ERRORED:
for (auto cmd: expandedPart) { case RunState::KILLED:
newExpandedPart.push_back(globalSub(cmd, param, val)); dag.setVertexState(update.taskName, RunState::RUNNING);
} dag.setVertexState(update.taskName, RunState::COMPLETED);
} break;
case RunState::COMPLETED:
case RunState::QUEUED:
break;
}
}
return dag;
}
TaskDAG runDAG(DAGRunID runID, executors::task::TaskExecutor &executor,
loggers::dag_run::DAGRunLogger &logger, TaskDAG dag,
const ConfigValues parameters)
{
logger.updateDAGRunState(runID, RunState::RUNNING);
std::unordered_map<std::string, std::future<AttemptRecord>> runningTasks;
std::unordered_map<std::string, size_t> taskAttemptCounts;
size_t running = 0;
size_t errored = 0;
while (!dag.allVisited()) {
// Check for any completed tasks
for (auto &[taskName, fut] : runningTasks) {
if (fut.valid()) {
auto attempt = fut.get();
logger.logTaskAttempt(runID, taskName, attempt);
auto &vert = dag.getVertex(taskName);
auto &task = vert.data;
if (attempt.rc == 0) {
logger.updateTaskState(runID, taskName, RunState::COMPLETED);
if (task.isGenerator) {
// Parse the output and update the DAGs
try {
auto newTasks = expandTaskSet(tasksFromJSON(attempt.outputLog),
executor, parameters);
updateDAGFromTasks(dag, newTasks);
for (const auto &[ntName, ntTask] : newTasks) {
logger.addTask(runID, ntName, ntTask);
dag.addEdge(taskName, ntName);
task.children.insert(ntName);
} }
logger.updateTask(runID, taskName, task);
expandedPart.swap(newExpandedPart); }
catch (std::exception &e) {
logger.logTaskAttempt(
runID, taskName,
AttemptRecord{
.executorLog =
std::string{"Failed to parse JSON output: "} +
e.what()});
logger.updateTaskState(runID, taskName, RunState::ERRORED);
++errored;
}
} }
dag.completeVisit(taskName);
std::vector<std::vector<std::string>> newCommands; --running;
for (const auto &newPart: expandedPart) { }
for (auto cmd: cooked) { else {
cmd.push_back(newPart); // RC isn't 0
newCommands.emplace_back(cmd); if (taskAttemptCounts[taskName] <= task.maxRetries) {
} logger.updateTaskState(runID, taskName, RunState::RETRY);
runningTasks[taskName] = executor.execute(taskName, task);
++taskAttemptCounts[taskName];
} }
cooked.swap(newCommands); else {
logger.updateTaskState(runID, taskName, RunState::ERRORED);
++errored;
}
}
} }
return cooked; }
// Add all remaining tasks in a task queue to avoid dominating the thread
// pool
auto t = dag.visitNext();
while (t.has_value()) {
// Schedule the task to run
auto &taskName = t.value().first;
auto &task = t.value().second;
taskAttemptCounts[taskName] = 1;
logger.updateTaskState(runID, taskName, RunState::RUNNING);
runningTasks.emplace(taskName, executor.execute(taskName, task));
++running;
auto nextTask = dag.visitNext();
if (not nextTask.has_value())
break;
t.emplace(nextTask.value());
}
if (running > 0 and errored == running) {
logger.updateDAGRunState(runID, RunState::ERRORED);
break;
}
std::this_thread::sleep_for(250ms);
} }
TaskSet if (dag.allVisited()) {
expandTaskSet(const TaskSet &tasks, logger.updateDAGRunState(runID, RunState::COMPLETED);
executors::task::TaskExecutor &executor,
const ConfigValues &interpolatedValues) {
// Expand the tasks first
TaskSet newTaskSet;
for (const auto &[baseName, task]: tasks) {
executor.validateTaskParameters(task.job);
const auto newJobs = executor.expandTaskParameters(task.job, interpolatedValues);
size_t i = 0;
for (const auto &newJob: newJobs) {
Task newTask{task};
newTask.job = newJob;
newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask);
++i;
}
}
return newTaskSet;
} }
return dag;
}
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks) { std::ostream &operator<<(std::ostream &os, const TimePoint &tp)
// Add the missing vertices {
for (const auto &[name, task]: tasks) { auto t_c = Clock::to_time_t(tp);
dag.addVertex(name, task); os << std::put_time(std::localtime(&t_c), "%Y-%m-%d %H:%M:%S %Z");
} return os;
}
// Add edges } // namespace daggy
for (const auto &[name, task]: tasks) {
dag.addEdgeIf(name, [&](const auto &v) { return task.children.count(v.data.definedName) > 0; });
}
if (! dag.isValid()) {
throw std::runtime_error("DAG contains a cycle");
}
}
TaskDAG buildDAGFromTasks(TaskSet &tasks,
const std::vector<loggers::dag_run::TaskUpdateRecord> &updates) {
TaskDAG dag;
updateDAGFromTasks(dag, tasks);
// Replay any updates
for (const auto &update: updates) {
switch (update.newState) {
case RunState::RUNNING:
case RunState::RETRY:
case RunState::ERRORED:
case RunState::KILLED:
dag.setVertexState(update.taskName, RunState::RUNNING);
dag.setVertexState(update.taskName, RunState::COMPLETED);
break;
case RunState::COMPLETED:
case RunState::QUEUED:
break;
}
}
return dag;
}
TaskDAG runDAG(DAGRunID runID,
executors::task::TaskExecutor &executor,
loggers::dag_run::DAGRunLogger &logger,
TaskDAG dag,
const ConfigValues parameters
) {
logger.updateDAGRunState(runID, RunState::RUNNING);
std::unordered_map<std::string, std::future<AttemptRecord>> runningTasks;
std::unordered_map<std::string, size_t> taskAttemptCounts;
size_t running = 0;
size_t errored = 0;
while (!dag.allVisited()) {
// Check for any completed tasks
for (auto &[taskName, fut]: runningTasks) {
if (fut.valid()) {
auto attempt = fut.get();
logger.logTaskAttempt(runID, taskName, attempt);
auto &vert = dag.getVertex(taskName);
auto &task = vert.data;
if (attempt.rc == 0) {
logger.updateTaskState(runID, taskName, RunState::COMPLETED);
if (task.isGenerator) {
// Parse the output and update the DAGs
try {
auto newTasks = expandTaskSet(tasksFromJSON(attempt.outputLog),
executor,
parameters
);
updateDAGFromTasks(dag, newTasks);
for (const auto &[ntName, ntTask]: newTasks) {
logger.addTask(runID, ntName, ntTask);
dag.addEdge(taskName, ntName);
task.children.insert(ntName);
}
logger.updateTask(runID, taskName, task);
} catch (std::exception &e) {
logger.logTaskAttempt(runID, taskName,
AttemptRecord{.executorLog =
std::string{"Failed to parse JSON output: "} +
e.what()});
logger.updateTaskState(runID, taskName, RunState::ERRORED);
++errored;
}
}
dag.completeVisit(taskName);
--running;
} else {
// RC isn't 0
if (taskAttemptCounts[taskName] <= task.maxRetries) {
logger.updateTaskState(runID, taskName, RunState::RETRY);
runningTasks[taskName] = executor.execute(taskName, task);
++taskAttemptCounts[taskName];
} else {
logger.updateTaskState(runID, taskName, RunState::ERRORED);
++errored;
}
}
}
}
// Add all remaining tasks in a task queue to avoid dominating the thread pool
auto t = dag.visitNext();
while (t.has_value()) {
// Schedule the task to run
auto &taskName = t.value().first;
auto &task = t.value().second;
taskAttemptCounts[taskName] = 1;
logger.updateTaskState(runID, taskName, RunState::RUNNING);
runningTasks.emplace(taskName, executor.execute(taskName, task));
++running;
auto nextTask = dag.visitNext();
if (not nextTask.has_value()) break;
t.emplace(nextTask.value());
}
if (running > 0 and errored == running) {
logger.updateDAGRunState(runID, RunState::ERRORED);
break;
}
std::this_thread::sleep_for(250ms);
}
if (dag.allVisited()) {
logger.updateDAGRunState(runID, RunState::COMPLETED);
}
return dag;
}
std::ostream &operator<<(std::ostream &os, const TimePoint &tp) {
auto t_c = Clock::to_time_t(tp);
os << std::put_time(std::localtime(&t_c), "%Y-%m-%d %H:%M:%S %Z");
return os;
}
}

202
daggy/src/executors/task/ForkingTaskExecutor.cpp
View File

@@ -1,122 +1,140 @@
#include <daggy/executors/task/ForkingTaskExecutor.hpp>
#include <daggy/Utilities.hpp>
#include <fcntl.h> #include <fcntl.h>
#include <poll.h>
#include <unistd.h> #include <unistd.h>
#include <wait.h> #include <wait.h>
#include <poll.h>
#include <daggy/Utilities.hpp>
#include <daggy/executors/task/ForkingTaskExecutor.hpp>
using namespace daggy::executors::task; using namespace daggy::executors::task;
std::string slurp(int fd) { std::string slurp(int fd)
std::string result; {
std::string result;
const ssize_t BUFFER_SIZE = 4096; const ssize_t BUFFER_SIZE = 4096;
char buffer[BUFFER_SIZE]; char buffer[BUFFER_SIZE];
struct pollfd pfd{.fd = fd, .events = POLLIN, .revents = 0}; struct pollfd pfd
{
.fd = fd, .events = POLLIN, .revents = 0
};
poll(&pfd, 1, 1);
while (pfd.revents & POLLIN) {
ssize_t bytes = read(fd, buffer, BUFFER_SIZE);
if (bytes == 0) {
break;
}
else {
result.append(buffer, bytes);
}
pfd.revents = 0;
poll(&pfd, 1, 1); poll(&pfd, 1, 1);
}
while (pfd.revents & POLLIN) { return result;
ssize_t bytes = read(fd, buffer, BUFFER_SIZE);
if (bytes == 0) {
break;
} else {
result.append(buffer, bytes);
}
pfd.revents = 0;
poll(&pfd, 1, 1);
}
return result;
} }
std::future<daggy::AttemptRecord> ForkingTaskExecutor::execute(
std::future<daggy::AttemptRecord> const std::string &taskName, const Task &task)
ForkingTaskExecutor::execute(const std::string &taskName, const Task &task) { {
return tp_.addTask([this, task](){return this->runTask(task);}); return tp_.addTask([this, task]() { return this->runTask(task); });
} }
daggy::AttemptRecord daggy::AttemptRecord ForkingTaskExecutor::runTask(const Task &task)
ForkingTaskExecutor::runTask(const Task & task) { {
AttemptRecord rec; AttemptRecord rec;
rec.startTime = Clock::now(); rec.startTime = Clock::now();
// Need to convert the strings // Need to convert the strings
std::vector<char *> argv; std::vector<char *> argv;
const auto command = std::get<Command>(task.job.at("command")); const auto command = std::get<Command>(task.job.at("command"));
std::transform(command.begin(), std::transform(
command.end(), command.begin(), command.end(), std::back_inserter(argv),
std::back_inserter(argv), [](const std::string &s) { return const_cast<char *>(s.c_str()); });
[](const std::string & s) { argv.push_back(nullptr);
return const_cast<char *>(s.c_str());
});
argv.push_back(nullptr);
// Create the pipe // Create the pipe
int stdoutPipe[2]; int stdoutPipe[2];
int pipeRC = pipe2(stdoutPipe, O_DIRECT); int pipeRC = pipe2(stdoutPipe, O_DIRECT);
if (pipeRC != 0) throw std::runtime_error("Unable to create pipe for stdout"); if (pipeRC != 0)
int stderrPipe[2]; throw std::runtime_error("Unable to create pipe for stdout");
pipeRC = pipe2(stderrPipe, O_DIRECT); int stderrPipe[2];
if (pipeRC != 0) throw std::runtime_error("Unable to create pipe for stderr"); pipeRC = pipe2(stderrPipe, O_DIRECT);
if (pipeRC != 0)
throw std::runtime_error("Unable to create pipe for stderr");
pid_t child = fork(); pid_t child = fork();
if (child < 0) { if (child < 0) {
throw std::runtime_error("Unable to fork child"); throw std::runtime_error("Unable to fork child");
} else if (child == 0) { // child }
while ((dup2(stdoutPipe[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {} else if (child == 0) { // child
while ((dup2(stderrPipe[1], STDERR_FILENO) == -1) && (errno == EINTR)) {} while ((dup2(stdoutPipe[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {
close(stdoutPipe[0]);
close(stderrPipe[0]);
execvp(argv[0], argv.data());
exit(-1);
} }
while ((dup2(stderrPipe[1], STDERR_FILENO) == -1) && (errno == EINTR)) {
std::atomic<bool> running = true;
std::thread stdoutReader([&]() { while (running) rec.outputLog.append(slurp(stdoutPipe[0])); });
std::thread stderrReader([&]() { while (running) rec.errorLog.append(slurp(stderrPipe[0])); });
int rc = 0;
waitpid(child, &rc, 0);
running = false;
rec.stopTime = Clock::now();
if (WIFEXITED(rc)) {
rec.rc = WEXITSTATUS(rc);
} else {
rec.rc = -1;
} }
stdoutReader.join();
stderrReader.join();
close(stdoutPipe[0]); close(stdoutPipe[0]);
close(stderrPipe[0]); close(stderrPipe[0]);
execvp(argv[0], argv.data());
exit(-1);
}
return rec; std::atomic<bool> running = true;
std::thread stdoutReader([&]() {
while (running)
rec.outputLog.append(slurp(stdoutPipe[0]));
});
std::thread stderrReader([&]() {
while (running)
rec.errorLog.append(slurp(stderrPipe[0]));
});
int rc = 0;
waitpid(child, &rc, 0);
running = false;
rec.stopTime = Clock::now();
if (WIFEXITED(rc)) {
rec.rc = WEXITSTATUS(rc);
}
else {
rec.rc = -1;
}
stdoutReader.join();
stderrReader.join();
close(stdoutPipe[0]);
close(stderrPipe[0]);
return rec;
} }
bool ForkingTaskExecutor::validateTaskParameters(const ConfigValues &job) { bool ForkingTaskExecutor::validateTaskParameters(const ConfigValues &job)
auto it = job.find("command"); {
if (it == job.end()) auto it = job.find("command");
throw std::runtime_error(R"(job does not have a "command" argument)"); if (it == job.end())
if (!std::holds_alternative<Command>(it->second)) throw std::runtime_error(R"(job does not have a "command" argument)");
throw std::runtime_error(R"(taskParameter's "command" must be an array of strings)"); if (!std::holds_alternative<Command>(it->second))
return true; throw std::runtime_error(
R"(taskParameter's "command" must be an array of strings)");
return true;
} }
std::vector<daggy::ConfigValues> std::vector<daggy::ConfigValues> ForkingTaskExecutor::expandTaskParameters(
ForkingTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) { const ConfigValues &job, const ConfigValues &expansionValues)
std::vector<ConfigValues> newValues; {
std::vector<ConfigValues> newValues;
const auto command = std::get<Command>(job.at("command")); const auto command = std::get<Command>(job.at("command"));
for (const auto &expandedCommand: interpolateValues(command, expansionValues)) { for (const auto &expandedCommand :
ConfigValues newCommand{job}; interpolateValues(command, expansionValues)) {
newCommand.at("command") = expandedCommand; ConfigValues newCommand{job};
newValues.emplace_back(newCommand); newCommand.at("command") = expandedCommand;
} newValues.emplace_back(newCommand);
}
return newValues; return newValues;
} }

75
daggy/src/executors/task/NoopTaskExecutor.cpp
View File

@@ -1,42 +1,45 @@
#include <daggy/executors/task/NoopTaskExecutor.hpp>
#include <daggy/Utilities.hpp> #include <daggy/Utilities.hpp>
#include <daggy/executors/task/NoopTaskExecutor.hpp>
namespace daggy::executors::task { namespace daggy::executors::task {
std::future<daggy::AttemptRecord> std::future<daggy::AttemptRecord> NoopTaskExecutor::execute(
NoopTaskExecutor::execute(const std::string &taskName, const Task &task) { const std::string &taskName, const Task &task)
std::promise<daggy::AttemptRecord> promise; {
auto ts = Clock::now(); std::promise<daggy::AttemptRecord> promise;
promise.set_value(AttemptRecord{ auto ts = Clock::now();
.startTime = ts, promise.set_value(AttemptRecord{.startTime = ts,
.stopTime = ts, .stopTime = ts,
.rc = 0, .rc = 0,
.executorLog = taskName, .executorLog = taskName,
.outputLog = taskName, .outputLog = taskName,
.errorLog = taskName .errorLog = taskName});
}); return promise.get_future();
return promise.get_future(); }
}
bool NoopTaskExecutor::validateTaskParameters(const ConfigValues &job) { bool NoopTaskExecutor::validateTaskParameters(const ConfigValues &job)
auto it = job.find("command"); {
if (it == job.end()) auto it = job.find("command");
throw std::runtime_error(R"(job does not have a "command" argument)"); if (it == job.end())
if (!std::holds_alternative<Command>(it->second)) throw std::runtime_error(R"(job does not have a "command" argument)");
throw std::runtime_error(R"(taskParameter's "command" must be an array of strings)"); if (!std::holds_alternative<Command>(it->second))
return true; throw std::runtime_error(
R"(taskParameter's "command" must be an array of strings)");
return true;
}
std::vector<daggy::ConfigValues> NoopTaskExecutor::expandTaskParameters(
const ConfigValues &job, const ConfigValues &expansionValues)
{
std::vector<ConfigValues> newValues;
const auto command = std::get<Command>(job.at("command"));
for (const auto &expandedCommand :
interpolateValues(command, expansionValues)) {
ConfigValues newCommand{job};
newCommand.at("command") = expandedCommand;
newValues.emplace_back(newCommand);
} }
std::vector<daggy::ConfigValues> return newValues;
NoopTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) { }
std::vector<ConfigValues> newValues; } // namespace daggy::executors::task
const auto command = std::get<Command>(job.at("command"));
for (const auto &expandedCommand: interpolateValues(command, expansionValues)) {
ConfigValues newCommand{job};
newCommand.at("command") = expandedCommand;
newValues.emplace_back(newCommand);
}
return newValues;
}
}

492
daggy/src/executors/task/SlurmTaskExecutor.cpp
View File

@@ -1,274 +1,274 @@
#include <iterator> #include <iterator>
#include <stdexcept> #include <stdexcept>
#ifdef DAGGY_ENABLE_SLURM #ifdef DAGGY_ENABLE_SLURM
#include <random> #include <slurm/slurm.h>
#include <stdlib.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <daggy/Utilities.hpp>
#include <daggy/executors/task/SlurmTaskExecutor.hpp>
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <random>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <slurm/slurm.h>
#include <daggy/executors/task/SlurmTaskExecutor.hpp>
#include <daggy/Utilities.hpp>
namespace fs = std::filesystem; namespace fs = std::filesystem;
namespace daggy::executors::task { namespace daggy::executors::task {
std::string getUniqueTag(size_t nChars = 6) { std::string getUniqueTag(size_t nChars = 6)
std::string result(nChars, '\0'); {
static std::random_device dev; std::string result(nChars, '\0');
static std::mt19937 rng(dev()); static std::random_device dev;
static std::mt19937 rng(dev());
std::uniform_int_distribution<int> dist(0, 61); std::uniform_int_distribution<int> dist(0, 61);
const char *v = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; const char *v =
"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
for (size_t i = 0; i < nChars; i++) { for (size_t i = 0; i < nChars; i++) {
result[i] = v[dist(rng)]; result[i] = v[dist(rng)];
} }
return result; return result;
}
void readAndClean(const fs::path &fn, std::string &dest)
{
if (!fs::exists(fn))
return;
std::ifstream ifh;
ifh.open(fn);
std::string contents(std::istreambuf_iterator<char>{ifh}, {});
ifh.close();
fs::remove_all(fn);
dest.swap(contents);
}
SlurmTaskExecutor::SlurmTaskExecutor()
: running_(true)
, monitorWorker_(&SlurmTaskExecutor::monitor, this)
{
std::string priority =
"SLURM_PRIO_PROCESS=" + std::to_string(getpriority(PRIO_PROCESS, 0));
std::string submitDir = "SLURM_SUBMIT_DIR=" + fs::current_path().string();
const size_t MAX_HOSTNAME_LENGTH = 50;
std::string submitHost(MAX_HOSTNAME_LENGTH, '\0');
gethostname(submitHost.data(), MAX_HOSTNAME_LENGTH);
submitHost = "SLURM_SUBMIT_HOST=" + submitHost;
submitHost.resize(submitHost.find('\0'));
uint32_t mask = umask(0);
umask(mask); // Restore the old mask
std::stringstream ss;
ss << "SLURM_UMASK=0" << uint32_t{((mask >> 6) & 07)}
<< uint32_t{((mask >> 3) & 07)} << uint32_t{(mask & 07)};
// Set some environment variables
putenv(const_cast<char *>(priority.c_str()));
putenv(const_cast<char *>(submitDir.c_str()));
putenv(const_cast<char *>(submitHost.c_str()));
putenv(const_cast<char *>(ss.str().c_str()));
}
SlurmTaskExecutor::~SlurmTaskExecutor()
{
running_ = false;
monitorWorker_.join();
}
// Validates the job to ensure that all required values are set and are of the
// right type,
bool SlurmTaskExecutor::validateTaskParameters(const ConfigValues &job)
{
const std::unordered_set<std::string> requiredFields{
"minCPUs", "minMemoryMB", "minTmpDiskMB",
"priority", "timeLimitSeconds", "userID",
"workDir", "tmpDir", "command"};
for (const auto &requiredField : requiredFields) {
if (job.count(requiredField) == 0) {
throw std::runtime_error("Missing field " + requiredField);
}
}
return true;
}
std::vector<ConfigValues> SlurmTaskExecutor::expandTaskParameters(
const ConfigValues &job, const ConfigValues &expansionValues)
{
std::vector<ConfigValues> newValues;
const auto command = std::get<Command>(job.at("command"));
for (const auto &expandedCommand :
interpolateValues(command, expansionValues)) {
ConfigValues newCommand{job};
newCommand.at("command") = expandedCommand;
newValues.emplace_back(newCommand);
} }
void readAndClean(const fs::path & fn, std::string & dest) { return newValues;
if (! fs::exists(fn)) return; }
std::ifstream ifh; std::future<AttemptRecord> SlurmTaskExecutor::execute(
ifh.open(fn); const std::string &taskName, const Task &task)
std::string contents(std::istreambuf_iterator<char>{ifh}, {}); {
ifh.close(); std::stringstream executorLog;
fs::remove_all(fn);
dest.swap(contents); const auto &job = task.job;
const auto uniqueTaskName = taskName + "_" + getUniqueTag(6);
fs::path tmpDir = std::get<std::string>(job.at("tmpDir"));
std::string stdoutFile = (tmpDir / (uniqueTaskName + ".stdout")).string();
std::string stderrFile = (tmpDir / (uniqueTaskName + ".stderr")).string();
std::string workDir = std::get<std::string>(job.at("workDir"));
// Convert command to argc / argv
std::vector<char *> argv{nullptr};
const auto command =
std::get<std::vector<std::string>>(task.job.at("command"));
std::transform(
command.begin(), command.end(), std::back_inserter(argv),
[](const std::string &s) { return const_cast<char *>(s.c_str()); });
char empty[] = "";
char *env[1];
env[0] = empty;
char script[] = "#!/bin/bash\n$@\n";
char stdinFile[] = "/dev/null";
// taken from slurm
int error_code;
job_desc_msg_t jd;
submit_response_msg_t *resp_msg;
slurm_init_job_desc_msg(&jd);
jd.contiguous = 1;
jd.name = const_cast<char *>(taskName.c_str());
jd.min_cpus = std::stoi(std::get<std::string>(job.at("minCPUs")));
jd.pn_min_memory = std::stoi(std::get<std::string>(job.at("minMemoryMB")));
jd.pn_min_tmp_disk =
std::stoi(std::get<std::string>(job.at("minTmpDiskMB")));
jd.priority = std::stoi(std::get<std::string>(job.at("priority")));
jd.shared = 0;
jd.time_limit =
std::stoi(std::get<std::string>(job.at("timeLimitSeconds")));
jd.min_nodes = 1;
jd.user_id = std::stoi(std::get<std::string>(job.at("userID")));
jd.argv = argv.data();
jd.argc = argv.size();
// TODO figure out the script to run
jd.script = script;
jd.std_in = stdinFile;
jd.std_err = const_cast<char *>(stderrFile.c_str());
jd.std_out = const_cast<char *>(stdoutFile.c_str());
jd.work_dir = const_cast<char *>(workDir.c_str());
jd.env_size = 1;
jd.environment = env;
/* TODO: Add support for environment
jobDescription.env_size = 2;
env[0] = "SLURM_ENV_0=looking_good";
env[1] = "SLURM_ENV_1=still_good";
jobDescription.environment = env;
*/
error_code = slurm_submit_batch_job(&jd, &resp_msg);
if (error_code) {
std::stringstream ss;
ss << "Unable to submit slurm job: " << slurm_strerror(error_code);
throw std::runtime_error(ss.str());
} }
SlurmTaskExecutor::SlurmTaskExecutor() uint32_t jobID = resp_msg->job_id;
: running_(true) executorLog << "Job " << resp_msg->job_submit_user_msg << '\n';
, monitorWorker_(&SlurmTaskExecutor::monitor, this) slurm_free_submit_response_response_msg(resp_msg);
{
std::string priority = "SLURM_PRIO_PROCESS=" + std::to_string(getpriority(PRIO_PROCESS, 0));
std::string submitDir = "SLURM_SUBMIT_DIR=" + fs::current_path().string();
const size_t MAX_HOSTNAME_LENGTH = 50; std::lock_guard<std::mutex> lock(promiseGuard_);
std::string submitHost(MAX_HOSTNAME_LENGTH, '\0'); Job newJob{.prom{}, .stdoutFile = stdoutFile, .stderrFile = stderrFile};
gethostname(submitHost.data(), MAX_HOSTNAME_LENGTH); auto fut = newJob.prom.get_future();
submitHost = "SLURM_SUBMIT_HOST=" + submitHost; runningJobs_.emplace(jobID, std::move(newJob));
submitHost.resize(submitHost.find('\0'));
uint32_t mask = umask(0); return fut;
umask(mask); // Restore the old mask }
std::stringstream ss;
ss << "SLURM_UMASK=0"
<< uint32_t{((mask >> 6) & 07)}
<< uint32_t{((mask >> 3) & 07)}
<< uint32_t{(mask & 07)};
// Set some environment variables
putenv(const_cast<char *>(priority.c_str()));
putenv(const_cast<char *>(submitDir.c_str()));
putenv(const_cast<char *>(submitHost.c_str()));
putenv(const_cast<char *>(ss.str().c_str()));
}
SlurmTaskExecutor::~SlurmTaskExecutor() {
running_ = false;
monitorWorker_.join();
}
// Validates the job to ensure that all required values are set and are of the right type,
bool SlurmTaskExecutor::validateTaskParameters(const ConfigValues &job) {
const std::unordered_set<std::string> requiredFields{
"minCPUs",
"minMemoryMB",
"minTmpDiskMB",
"priority",
"timeLimitSeconds",
"userID",
"workDir",
"tmpDir",
"command"
};
for (const auto &requiredField: requiredFields) {
if (job.count(requiredField) == 0) {
throw std::runtime_error("Missing field " + requiredField);
}
}
return true;
}
std::vector<ConfigValues>
SlurmTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) {
std::vector<ConfigValues> newValues;
const auto command = std::get<Command>(job.at("command"));
for (const auto &expandedCommand: interpolateValues(command, expansionValues)) {
ConfigValues newCommand{job};
newCommand.at("command") = expandedCommand;
newValues.emplace_back(newCommand);
}
return newValues;
}
std::future<AttemptRecord>
SlurmTaskExecutor::execute(const std::string &taskName, const Task &task) {
std::stringstream executorLog;
const auto &job = task.job;
const auto uniqueTaskName = taskName + "_" + getUniqueTag(6);
fs::path tmpDir = std::get<std::string>(job.at("tmpDir"));
std::string stdoutFile = (tmpDir / (uniqueTaskName + ".stdout")).string();
std::string stderrFile = (tmpDir / (uniqueTaskName + ".stderr")).string();
std::string workDir = std::get<std::string>(job.at("workDir"));
// Convert command to argc / argv
std::vector<char *> argv{nullptr};
const auto command = std::get<std::vector<std::string>>(task.job.at("command"));
std::transform(command.begin(),
command.end(),
std::back_inserter(argv),
[](const std::string & s) {
return const_cast<char *>(s.c_str());
});
char empty[] = "";
char *env[1];
env[0] = empty;
char script[] = "#!/bin/bash\n$@\n";
char stdinFile[] = "/dev/null";
// taken from slurm
int error_code;
job_desc_msg_t jd;
submit_response_msg_t *resp_msg;
slurm_init_job_desc_msg(&jd);
jd.contiguous = 1;
jd.name = const_cast<char *>(taskName.c_str());
jd.min_cpus = std::stoi(std::get<std::string>(job.at("minCPUs")));
jd.pn_min_memory = std::stoi(std::get<std::string>(job.at("minMemoryMB")));
jd.pn_min_tmp_disk = std::stoi(std::get<std::string>(job.at("minTmpDiskMB")));
jd.priority = std::stoi(std::get<std::string>(job.at("priority")));
jd.shared = 0;
jd.time_limit = std::stoi(std::get<std::string>(job.at("timeLimitSeconds")));
jd.min_nodes = 1;
jd.user_id = std::stoi(std::get<std::string>(job.at("userID")));
jd.argv = argv.data();
jd.argc = argv.size();
// TODO figure out the script to run
jd.script = script;
jd.std_in = stdinFile;
jd.std_err = const_cast<char *>(stderrFile.c_str());
jd.std_out = const_cast<char *>(stdoutFile.c_str());
jd.work_dir = const_cast<char *>(workDir.c_str());
jd.env_size = 1;
jd.environment = env;
/* TODO: Add support for environment
jobDescription.env_size = 2;
env[0] = "SLURM_ENV_0=looking_good";
env[1] = "SLURM_ENV_1=still_good";
jobDescription.environment = env;
*/
error_code = slurm_submit_batch_job(&jd, &resp_msg);
if (error_code) {
std::stringstream ss;
ss << "Unable to submit slurm job: "
<< slurm_strerror(error_code);
throw std::runtime_error(ss.str());
}
uint32_t jobID = resp_msg->job_id;
executorLog << "Job " << resp_msg->job_submit_user_msg << '\n';
slurm_free_submit_response_response_msg(resp_msg);
void SlurmTaskExecutor::monitor()
{
std::unordered_set<size_t> resolvedJobs;
while (running_) {
{
std::lock_guard<std::mutex> lock(promiseGuard_); std::lock_guard<std::mutex> lock(promiseGuard_);
Job newJob{ for (auto &[jobID, job] : runningJobs_) {
.prom{}, job_info_msg_t *jobStatus;
.stdoutFile = stdoutFile, int error_code =
.stderrFile = stderrFile slurm_load_job(&jobStatus, jobID, SHOW_ALL | SHOW_DETAIL);
}; if (error_code != SLURM_SUCCESS)
auto fut = newJob.prom.get_future(); continue;
runningJobs_.emplace(jobID, std::move(newJob));
return fut; uint32_t idx = jobStatus->record_count;
} if (idx == 0)
continue;
idx--;
const slurm_job_info_t &jobInfo = jobStatus->job_array[idx];
AttemptRecord record;
switch (jobInfo.job_state) {
case JOB_PENDING:
case JOB_SUSPENDED:
case JOB_RUNNING:
continue;
break;
// Job has finished
case JOB_COMPLETE: /* completed execution successfully */
case JOB_FAILED: /* completed execution unsuccessfully */
record.executorLog = "Script errored.\n";
break;
case JOB_CANCELLED: /* cancelled by user */
record.executorLog = "Job cancelled by user.\n";
break;
case JOB_TIMEOUT: /* terminated on reaching time limit */
record.executorLog = "Job exceeded time limit.\n";
break;
case JOB_NODE_FAIL: /* terminated on node failure */
record.executorLog = "Node failed during execution\n";
break;
case JOB_PREEMPTED: /* terminated due to preemption */
record.executorLog = "Job terminated due to pre-emption.\n";
break;
case JOB_BOOT_FAIL: /* terminated due to node boot failure */
record.executorLog =
"Job failed to run due to failure of compute node to boot.\n";
break;
case JOB_DEADLINE: /* terminated on deadline */
record.executorLog = "Job terminated due to deadline.\n";
break;
case JOB_OOM: /* experienced out of memory error */
record.executorLog = "Job terminated due to out-of-memory.\n";
break;
}
record.rc = jobInfo.exit_code;
slurm_free_job_info_msg(jobStatus);
void SlurmTaskExecutor::monitor() { readAndClean(job.stdoutFile, record.outputLog);
std::unordered_set<size_t> resolvedJobs; readAndClean(job.stderrFile, record.errorLog);
while (running_) {
{
std::lock_guard<std::mutex> lock(promiseGuard_);
for (auto & [jobID, job] : runningJobs_) {
job_info_msg_t * jobStatus;
int error_code = slurm_load_job(&jobStatus, jobID, SHOW_ALL | SHOW_DETAIL);
if (error_code != SLURM_SUCCESS) continue;
uint32_t idx = jobStatus->record_count; job.prom.set_value(std::move(record));
if (idx == 0) continue; resolvedJobs.insert(jobID);
idx--;
const slurm_job_info_t & jobInfo = jobStatus->job_array[idx];
AttemptRecord record;
switch(jobInfo.job_state) {
case JOB_PENDING:
case JOB_SUSPENDED:
case JOB_RUNNING:
continue;
break;
// Job has finished
case JOB_COMPLETE: /* completed execution successfully */
case JOB_FAILED: /* completed execution unsuccessfully */
record.executorLog = "Script errored.\n";
break;
case JOB_CANCELLED: /* cancelled by user */
record.executorLog = "Job cancelled by user.\n";
break;
case JOB_TIMEOUT: /* terminated on reaching time limit */
record.executorLog = "Job exceeded time limit.\n";
break;
case JOB_NODE_FAIL: /* terminated on node failure */
record.executorLog = "Node failed during execution\n";
break;
case JOB_PREEMPTED: /* terminated due to preemption */
record.executorLog = "Job terminated due to pre-emption.\n";
break;
case JOB_BOOT_FAIL: /* terminated due to node boot failure */
record.executorLog = "Job failed to run due to failure of compute node to boot.\n";
break;
case JOB_DEADLINE: /* terminated on deadline */
record.executorLog = "Job terminated due to deadline.\n";
break;
case JOB_OOM: /* experienced out of memory error */
record.executorLog = "Job terminated due to out-of-memory.\n";
break;
}
record.rc = jobInfo.exit_code;
slurm_free_job_info_msg(jobStatus);
readAndClean(job.stdoutFile, record.outputLog);
readAndClean(job.stderrFile, record.errorLog);
job.prom.set_value(std::move(record));
resolvedJobs.insert(jobID);
}
for (const auto &jobID : resolvedJobs) {
runningJobs_.extract(jobID);
}
}
std::this_thread::sleep_for(std::chrono::milliseconds(250));
} }
for (const auto &jobID : resolvedJobs) {
runningJobs_.extract(jobID);
}
}
std::this_thread::sleep_for(std::chrono::milliseconds(250));
} }
} }
} // namespace daggy::executors::task
#endif #endif

330
daggy/src/loggers/dag_run/FileSystemLogger.cpp
View File

@@ -1,178 +1,212 @@
#include <fstream>
#include <iomanip>
#include <enum.h> #include <enum.h>
#include <daggy/loggers/dag_run/FileSystemLogger.hpp>
#include <daggy/Serialization.hpp> #include <daggy/Serialization.hpp>
#include <daggy/Utilities.hpp> #include <daggy/Utilities.hpp>
#include <daggy/loggers/dag_run/FileSystemLogger.hpp>
#include <fstream>
#include <iomanip>
namespace fs = std::filesystem; namespace fs = std::filesystem;
using namespace daggy::loggers::dag_run; using namespace daggy::loggers::dag_run;
namespace daggy { namespace daggy {
inline const fs::path FileSystemLogger::getCurrentPath() const { return root_ / "current"; } inline const fs::path FileSystemLogger::getCurrentPath() const
{
return root_ / "current";
}
inline const fs::path FileSystemLogger::getRunsRoot() const { return root_ / "runs"; } inline const fs::path FileSystemLogger::getRunsRoot() const
{
return root_ / "runs";
}
inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const { inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const
return getRunsRoot() / std::to_string(runID); {
return getRunsRoot() / std::to_string(runID);
}
FileSystemLogger::FileSystemLogger(fs::path root)
: root_(root)
, nextRunID_(0)
{
const std::vector<fs::path> reqPaths{root_, getCurrentPath(),
getRunsRoot()};
for (const auto &path : reqPaths) {
if (!fs::exists(path)) {
fs::create_directories(path);
}
} }
FileSystemLogger::FileSystemLogger(fs::path root) // Get the next run ID
: root_(root), nextRunID_(0) { for (auto &dir : fs::directory_iterator(getRunsRoot())) {
const std::vector<fs::path> reqPaths{root_, getCurrentPath(), getRunsRoot()}; try {
for (const auto &path: reqPaths) { size_t runID = std::stoull(dir.path().stem());
if (!fs::exists(path)) { fs::create_directories(path); } if (runID > nextRunID_)
} nextRunID_ = runID + 1;
}
catch (std::exception &e) {
continue;
}
}
}
// Get the next run ID // Execution
for (auto &dir: fs::directory_iterator(getRunsRoot())) { DAGRunID FileSystemLogger::startDAGRun(std::string name, const TaskSet &tasks)
try { {
size_t runID = std::stoull(dir.path().stem()); DAGRunID runID = nextRunID_++;
if (runID > nextRunID_) nextRunID_ = runID + 1;
} catch (std::exception &e) { // TODO make this threadsafe
continue; fs::path runDir = getRunRoot(runID);
} // std::lock_guard<std::mutex> guard(runLocks[runDir]);
}
// Init the directory
fs::path runRoot = getRunsRoot() / std::to_string(runID);
fs::create_directories(runRoot);
// Create meta.json with DAGRun Name and task definitions
std::ofstream ofh(runRoot / "metadata.json",
std::ios::trunc | std::ios::binary);
ofh << R"({ "name": )" << std::quoted(name) << R"(, "tasks": )"
<< tasksToJSON(tasks) << "}\n";
ofh.close();
// Task directories
for (const auto &[name, task] : tasks) {
auto taskDir = runRoot / name;
fs::create_directories(taskDir);
std::ofstream ofh(taskDir / "states.csv");
} }
// Execution return runID;
DAGRunID FileSystemLogger::startDAGRun(std::string name, const TaskSet &tasks) { }
DAGRunID runID = nextRunID_++;
// TODO make this threadsafe void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state)
fs::path runDir = getRunRoot(runID); {
// std::lock_guard<std::mutex> guard(runLocks[runDir]); std::ofstream ofh(getRunRoot(dagRunID) / "states.csv",
std::ios::binary | std::ios::app);
ofh << std::quoted(timePointToString(Clock::now())) << ','
<< state._to_string() << '\n';
ofh.flush();
ofh.close();
}
// Init the directory void FileSystemLogger::logTaskAttempt(DAGRunID dagRunID,
fs::path runRoot = getRunsRoot() / std::to_string(runID); const std::string &taskName,
fs::create_directories(runRoot); const AttemptRecord &attempt)
{
// Create meta.json with DAGRun Name and task definitions auto taskRoot = getRunRoot(dagRunID) / taskName;
std::ofstream ofh(runRoot / "metadata.json", std::ios::trunc | std::ios::binary); size_t i = 1;
ofh << R"({ "name": )" << std::quoted(name) << R"(, "tasks": )" << tasksToJSON(tasks) << "}\n"; while (fs::exists(taskRoot / std::to_string(i))) {
ofh.close(); ++i;
// Task directories
for (const auto &[name, task]: tasks) {
auto taskDir = runRoot / name;
fs::create_directories(taskDir);
std::ofstream ofh(taskDir / "states.csv");
}
return runID;
} }
void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) { auto attemptDir = taskRoot / std::to_string(i);
std::ofstream ofh(getRunRoot(dagRunID) / "states.csv", std::ios::binary | std::ios::app); fs::create_directories(attemptDir);
ofh << std::quoted(timePointToString(Clock::now())) << ',' << state._to_string() << '\n';
ofh.flush(); std::ofstream ofh;
ofh.close();
// Metadata
ofh.open(attemptDir / "metadata.json");
ofh << "{\n"
<< R"("startTime": )"
<< std::quoted(timePointToString(attempt.startTime)) << ",\n"
<< R"("stopTime": )" << std::quoted(timePointToString(attempt.stopTime))
<< ",\n"
<< R"("rc": )" << attempt.rc << '\n'
<< '}';
// output
ofh.open(attemptDir / "executor.log");
ofh << attempt.executorLog << std::flush;
ofh.close();
// Output
ofh.open(attemptDir / "output.log");
ofh << attempt.outputLog << std::flush;
ofh.close();
// Error
ofh.open(attemptDir / "error.log");
ofh << attempt.errorLog << std::flush;
ofh.close();
}
void FileSystemLogger::updateTaskState(DAGRunID dagRunID,
const std::string &taskName,
RunState state)
{
std::ofstream ofh(getRunRoot(dagRunID) / taskName / "states.csv",
std::ios::binary | std::ios::app);
ofh << std::quoted(timePointToString(Clock::now())) << ','
<< state._to_string() << '\n';
ofh.flush();
ofh.close();
}
// Querying
std::vector<DAGRunSummary> FileSystemLogger::getDAGs(uint32_t stateMask)
{
return {};
}
DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunID)
{
DAGRunRecord record;
auto runRoot = getRunRoot(dagRunID);
if (!fs::exists(runRoot)) {
throw std::runtime_error("No DAGRun with that ID exists");
} }
void std::ifstream ifh(runRoot / "metadata.json", std::ios::binary);
FileSystemLogger::logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, std::string metaData;
const AttemptRecord &attempt) { std::getline(ifh, metaData, '\0');
auto taskRoot = getRunRoot(dagRunID) / taskName; ifh.close();
size_t i = 1;
while (fs::exists(taskRoot / std::to_string(i))) { ++i; }
auto attemptDir = taskRoot / std::to_string(i); rj::Document doc;
fs::create_directories(attemptDir); doc.Parse(metaData.c_str());
std::ofstream ofh; record.name = doc["name"].GetString();
record.tasks = tasksFromJSON(doc["tasks"]);
// Metadata // DAG State Changes
ofh.open(attemptDir / "metadata.json"); std::string line;
ofh << "{\n" std::string token;
<< R"("startTime": )" << std::quoted(timePointToString(attempt.startTime)) << ",\n" auto dagStateFile = runRoot / "states.csv";
<< R"("stopTime": )" << std::quoted(timePointToString(attempt.stopTime)) << ",\n" ifh.open(dagStateFile);
<< R"("rc": )" << attempt.rc << '\n' while (std::getline(ifh, line)) {
<< '}'; std::stringstream ss{line};
std::string time;
std::string state;
std::getline(ss, time, ',');
std::getline(ss, state);
// output record.dagStateChanges.emplace_back(
ofh.open(attemptDir / "executor.log"); DAGUpdateRecord{.time = stringToTimePoint(time),
ofh << attempt.executorLog << std::flush; .newState = RunState::_from_string(state.c_str())});
ofh.close();
// Output
ofh.open(attemptDir / "output.log");
ofh << attempt.outputLog << std::flush;
ofh.close();
// Error
ofh.open(attemptDir / "error.log");
ofh << attempt.errorLog << std::flush;
ofh.close();
} }
ifh.close();
void FileSystemLogger::updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) { // Task states
std::ofstream ofh(getRunRoot(dagRunID) / taskName / "states.csv", std::ios::binary | std::ios::app); for (const auto &[taskName, task] : record.tasks) {
ofh << std::quoted(timePointToString(Clock::now())) << ',' << state._to_string() << '\n'; auto taskStateFile = runRoot / taskName / "states.csv";
ofh.flush(); if (!fs::exists(taskStateFile)) {
ofh.close(); record.taskRunStates.emplace(taskName, RunState::QUEUED);
continue;
}
ifh.open(taskStateFile);
while (std::getline(ifh, line)) {
continue;
}
std::stringstream ss{line};
while (std::getline(ss, token, ',')) {
continue;
}
RunState taskState = RunState::_from_string(token.c_str());
record.taskRunStates.emplace(taskName, taskState);
ifh.close();
} }
return record;
// Querying }
std::vector<DAGRunSummary> FileSystemLogger::getDAGs(uint32_t stateMask) { } // namespace daggy
return {};
}
DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunID) {
DAGRunRecord record;
auto runRoot = getRunRoot(dagRunID);
if (!fs::exists(runRoot)) {
throw std::runtime_error("No DAGRun with that ID exists");
}
std::ifstream ifh(runRoot / "metadata.json", std::ios::binary);
std::string metaData;
std::getline(ifh, metaData, '\0');
ifh.close();
rj::Document doc;
doc.Parse(metaData.c_str());
record.name = doc["name"].GetString();
record.tasks = tasksFromJSON(doc["tasks"]);
// DAG State Changes
std::string line;
std::string token;
auto dagStateFile = runRoot / "states.csv";
ifh.open(dagStateFile);
while (std::getline(ifh, line)) {
std::stringstream ss{line};
std::string time;
std::string state;
std::getline(ss, time, ',');
std::getline(ss, state);
record.dagStateChanges.emplace_back(DAGUpdateRecord{
.time = stringToTimePoint(time),
.newState = RunState::_from_string(state.c_str())
});
}
ifh.close();
// Task states
for (const auto &[taskName, task]: record.tasks) {
auto taskStateFile = runRoot / taskName / "states.csv";
if (!fs::exists(taskStateFile)) {
record.taskRunStates.emplace(taskName, RunState::QUEUED);
continue;
}
ifh.open(taskStateFile);
while (std::getline(ifh, line)) { continue; }
std::stringstream ss{line};
while (std::getline(ss, token, ',')) { continue; }
RunState taskState = RunState::_from_string(token.c_str());
record.taskRunStates.emplace(taskName, taskState);
ifh.close();
}
return record;
}
}

245
daggy/src/loggers/dag_run/OStreamLogger.cpp
View File

@@ -1,122 +1,135 @@
#include <iterator>
#include <algorithm>
#include <enum.h> #include <enum.h>
#include <daggy/loggers/dag_run/OStreamLogger.hpp> #include <algorithm>
#include <daggy/Serialization.hpp> #include <daggy/Serialization.hpp>
#include <daggy/loggers/dag_run/OStreamLogger.hpp>
#include <iterator>
namespace daggy { namespace daggy { namespace loggers { namespace dag_run {
namespace loggers { OStreamLogger::OStreamLogger(std::ostream &os)
namespace dag_run { : os_(os)
OStreamLogger::OStreamLogger(std::ostream &os) : os_(os) {} {
}
// Execution // Execution
DAGRunID OStreamLogger::startDAGRun(std::string name, const TaskSet &tasks) { DAGRunID OStreamLogger::startDAGRun(std::string name, const TaskSet &tasks)
std::lock_guard<std::mutex> lock(guard_); {
size_t runID = dagRuns_.size(); std::lock_guard<std::mutex> lock(guard_);
dagRuns_.push_back({ size_t runID = dagRuns_.size();
.name = name, dagRuns_.push_back({.name = name, .tasks = tasks});
.tasks = tasks for (const auto &[name, _] : tasks) {
}); _updateTaskState(runID, name, RunState::QUEUED);
for (const auto &[name, _]: tasks) {
_updateTaskState(runID, name, RunState::QUEUED);
}
_updateDAGRunState(runID, RunState::QUEUED);
os_ << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size()
<< " tasks" << std::endl;
for (const auto &[name, task]: tasks) {
os_ << "TASK (" << name << "): " << configToJSON(task.job);
os_ << std::endl;
}
return runID;
}
void OStreamLogger::addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) {
std::lock_guard<std::mutex> lock(guard_);
auto &dagRun = dagRuns_[dagRunID];
dagRun.tasks[taskName] = task;
_updateTaskState(dagRunID, taskName, RunState::QUEUED);
}
void OStreamLogger::updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) {
std::lock_guard<std::mutex> lock(guard_);
auto &dagRun = dagRuns_[dagRunID];
dagRun.tasks[taskName] = task;
}
void OStreamLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) {
std::lock_guard<std::mutex> lock(guard_);
_updateDAGRunState(dagRunID, state);
}
void OStreamLogger::_updateDAGRunState(DAGRunID dagRunID, RunState state) {
os_ << "DAG State Change(" << dagRunID << "): " << state._to_string() << std::endl;
dagRuns_[dagRunID].dagStateChanges.push_back({Clock::now(), state});
}
void OStreamLogger::logTaskAttempt(DAGRunID dagRunID, const std::string &taskName,
const AttemptRecord &attempt) {
std::lock_guard<std::mutex> lock(guard_);
const std::string &msg = attempt.rc == 0 ? attempt.outputLog : attempt.errorLog;
os_ << "Task Attempt (" << dagRunID << '/' << taskName << "): Ran with RC " << attempt.rc << ": "
<< msg << std::endl;
dagRuns_[dagRunID].taskAttempts[taskName].push_back(attempt);
}
void OStreamLogger::updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) {
std::lock_guard<std::mutex> lock(guard_);
_updateTaskState(dagRunID, taskName, state);
}
void OStreamLogger::_updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) {
auto &dagRun = dagRuns_.at(dagRunID);
dagRun.taskStateChanges.push_back({Clock::now(), taskName, state});
auto it = dagRun.taskRunStates.find(taskName);
if (it == dagRun.taskRunStates.end()) {
dagRun.taskRunStates.emplace(taskName, state);
} else {
it->second = state;
}
os_ << "Task State Change (" << dagRunID << '/' << taskName << "): "
<< state._to_string()
<< std::endl;
}
// Querying
std::vector<DAGRunSummary> OStreamLogger::getDAGs(uint32_t stateMask) {
std::vector<DAGRunSummary> summaries;
std::lock_guard<std::mutex> lock(guard_);
size_t i = 0;
for (const auto &run: dagRuns_) {
DAGRunSummary summary{
.runID = i,
.name = run.name,
.runState = run.dagStateChanges.back().newState,
.startTime = run.dagStateChanges.front().time,
.lastUpdate = std::max<TimePoint>(run.taskStateChanges.back().time,
run.dagStateChanges.back().time)
};
for (const auto &[_, taskState]: run.taskRunStates) {
summary.taskStateCounts[taskState]++;
}
summaries.emplace_back(summary);
}
return summaries;
}
DAGRunRecord OStreamLogger::getDAGRun(DAGRunID dagRunID) {
if (dagRunID >= dagRuns_.size()) {
throw std::runtime_error("No such DAGRun ID");
}
std::lock_guard<std::mutex> lock(guard_);
return dagRuns_[dagRunID];
}
}
} }
} _updateDAGRunState(runID, RunState::QUEUED);
os_ << "Starting new DAGRun named " << name << " with ID " << runID
<< " and " << tasks.size() << " tasks" << std::endl;
for (const auto &[name, task] : tasks) {
os_ << "TASK (" << name << "): " << configToJSON(task.job);
os_ << std::endl;
}
return runID;
}
void OStreamLogger::addTask(DAGRunID dagRunID, const std::string taskName,
const Task &task)
{
std::lock_guard<std::mutex> lock(guard_);
auto &dagRun = dagRuns_[dagRunID];
dagRun.tasks[taskName] = task;
_updateTaskState(dagRunID, taskName, RunState::QUEUED);
}
void OStreamLogger::updateTask(DAGRunID dagRunID, const std::string taskName,
const Task &task)
{
std::lock_guard<std::mutex> lock(guard_);
auto &dagRun = dagRuns_[dagRunID];
dagRun.tasks[taskName] = task;
}
void OStreamLogger::updateDAGRunState(DAGRunID dagRunID, RunState state)
{
std::lock_guard<std::mutex> lock(guard_);
_updateDAGRunState(dagRunID, state);
}
void OStreamLogger::_updateDAGRunState(DAGRunID dagRunID, RunState state)
{
os_ << "DAG State Change(" << dagRunID << "): " << state._to_string()
<< std::endl;
dagRuns_[dagRunID].dagStateChanges.push_back({Clock::now(), state});
}
void OStreamLogger::logTaskAttempt(DAGRunID dagRunID,
const std::string &taskName,
const AttemptRecord &attempt)
{
std::lock_guard<std::mutex> lock(guard_);
const std::string &msg =
attempt.rc == 0 ? attempt.outputLog : attempt.errorLog;
os_ << "Task Attempt (" << dagRunID << '/' << taskName << "): Ran with RC "
<< attempt.rc << ": " << msg << std::endl;
dagRuns_[dagRunID].taskAttempts[taskName].push_back(attempt);
}
void OStreamLogger::updateTaskState(DAGRunID dagRunID,
const std::string &taskName,
RunState state)
{
std::lock_guard<std::mutex> lock(guard_);
_updateTaskState(dagRunID, taskName, state);
}
void OStreamLogger::_updateTaskState(DAGRunID dagRunID,
const std::string &taskName,
RunState state)
{
auto &dagRun = dagRuns_.at(dagRunID);
dagRun.taskStateChanges.push_back({Clock::now(), taskName, state});
auto it = dagRun.taskRunStates.find(taskName);
if (it == dagRun.taskRunStates.end()) {
dagRun.taskRunStates.emplace(taskName, state);
}
else {
it->second = state;
}
os_ << "Task State Change (" << dagRunID << '/' << taskName
<< "): " << state._to_string() << std::endl;
}
// Querying
std::vector<DAGRunSummary> OStreamLogger::getDAGs(uint32_t stateMask)
{
std::vector<DAGRunSummary> summaries;
std::lock_guard<std::mutex> lock(guard_);
size_t i = 0;
for (const auto &run : dagRuns_) {
DAGRunSummary summary{
.runID = i,
.name = run.name,
.runState = run.dagStateChanges.back().newState,
.startTime = run.dagStateChanges.front().time,
.lastUpdate = std::max<TimePoint>(run.taskStateChanges.back().time,
run.dagStateChanges.back().time)};
for (const auto &[_, taskState] : run.taskRunStates) {
summary.taskStateCounts[taskState]++;
}
summaries.emplace_back(summary);
}
return summaries;
}
DAGRunRecord OStreamLogger::getDAGRun(DAGRunID dagRunID)
{
if (dagRunID >= dagRuns_.size()) {
throw std::runtime_error("No such DAGRun ID");
}
std::lock_guard<std::mutex> lock(guard_);
return dagRuns_[dagRunID];
}
}}} // namespace daggy::loggers::dag_run

8
tests/int_basic.cpp
View File

@@ -1,9 +1,9 @@
#include <catch2/catch.hpp>
#include <iostream> #include <iostream>
#include "daggy/DAG.hpp" #include "daggy/DAG.hpp"
#include <catch2/catch.hpp> TEST_CASE("General tests", "[general]")
{
TEST_CASE("General tests", "[general]") { REQUIRE(1 == 1);
REQUIRE(1 == 1);
} }

5
tests/main.cpp
View File

@@ -6,8 +6,9 @@
#include <catch2/catch.hpp> #include <catch2/catch.hpp>
TEST_CASE("Sanity tests", "[sanity]") { TEST_CASE("Sanity tests", "[sanity]")
REQUIRE(1 == 1); {
REQUIRE(1 == 1);
} }
// compile and run // compile and run

135
tests/unit_dag.cpp
View File

@@ -1,90 +1,87 @@
#include <catch2/catch.hpp>
#include <iostream> #include <iostream>
#include "daggy/DAG.hpp" #include "daggy/DAG.hpp"
#include <catch2/catch.hpp> TEST_CASE("dag_construction", "[dag]")
{
daggy::DAG<size_t, size_t> dag;
TEST_CASE("dag_construction", "[dag]") { REQUIRE(dag.size() == 0);
daggy::DAG<size_t, size_t> dag; REQUIRE(dag.empty());
REQUIRE(dag.size() == 0); REQUIRE_NOTHROW(dag.addVertex(0, 0));
REQUIRE(dag.empty()); for (size_t i = 1; i < 10; ++i) {
dag.addVertex(i, i);
REQUIRE(dag.hasVertex(i));
REQUIRE(dag.getVertex(i).data == i);
dag.addEdge(i - 1, i);
}
REQUIRE_NOTHROW(dag.addVertex(0, 0)); REQUIRE(dag.size() == 10);
for (size_t i = 1; i < 10; ++i) { REQUIRE(!dag.empty());
dag.addVertex(i, i);
REQUIRE(dag.hasVertex(i));
REQUIRE(dag.getVertex(i).data == i);
dag.addEdge(i - 1, i);
}
REQUIRE(dag.size() == 10); // Cannot add an edge that would result in a cycle
REQUIRE(!dag.empty()); dag.addEdge(9, 5);
REQUIRE(!dag.isValid());
// Cannot add an edge that would result in a cycle // Bounds checking
dag.addEdge(9, 5); SECTION("addEdge Bounds Checking")
REQUIRE(!dag.isValid()); {
REQUIRE_THROWS(dag.addEdge(20, 0));
// Bounds checking REQUIRE_THROWS(dag.addEdge(0, 20));
SECTION("addEdge Bounds Checking") { }
REQUIRE_THROWS(dag.addEdge(20, 0));
REQUIRE_THROWS(dag.addEdge(0, 20));
}
} }
TEST_CASE("dag_traversal", "[dag]") { TEST_CASE("dag_traversal", "[dag]")
daggy::DAG<size_t, size_t> dag; {
daggy::DAG<size_t, size_t> dag;
const int N_VERTICES = 10; const int N_VERTICES = 10;
for (int i = 0; i < N_VERTICES; ++i) { dag.addVertex(i, i); } for (int i = 0; i < N_VERTICES; ++i) {
dag.addVertex(i, i);
}
/* /*
0 ---------------------\ 0 ---------------------\
1 ---------- \ \ /-----> 8 1 ---------- \ \ /-----> 8
2 ---- 3 ---- > 5 -------> 6 -----> 7 2 ---- 3 ---- > 5 -------> 6 -----> 7
4 -------------------------------/ \-----> 9 4 -------------------------------/ \-----> 9
*/ */
std::vector<std::pair<int, int>> edges{ std::vector<std::pair<int, int>> edges{{0, 6}, {1, 5}, {5, 6}, {6, 7}, {2, 3},
{0, 6}, {3, 5}, {4, 7}, {7, 8}, {7, 9}};
{1, 5},
{5, 6},
{6, 7},
{2, 3},
{3, 5},
{4, 7},
{7, 8},
{7, 9}
};
for (auto const[from, to]: edges) { for (auto const [from, to] : edges) {
dag.addEdge(from, to); dag.addEdge(from, to);
}
SECTION("Basic Traversal")
{
dag.reset();
std::vector<int> visitOrder(N_VERTICES);
size_t i = 0;
while (!dag.allVisited()) {
const auto v = dag.visitNext().value();
dag.completeVisit(v.first);
visitOrder[v.first] = i;
++i;
} }
SECTION("Basic Traversal") { // Ensure visit order is preserved
dag.reset(); for (auto const [from, to] : edges) {
std::vector<int> visitOrder(N_VERTICES); REQUIRE(visitOrder[from] <= visitOrder[to]);
size_t i = 0;
while (!dag.allVisited()) {
const auto v = dag.visitNext().value();
dag.completeVisit(v.first);
visitOrder[v.first] = i;
++i;
}
// Ensure visit order is preserved
for (auto const[from, to]: edges) {
REQUIRE(visitOrder[from] <= visitOrder[to]);
}
} }
}
SECTION("Iteration") { SECTION("Iteration")
size_t nVisited = 0; {
dag.forEach([&](auto &k) { size_t nVisited = 0;
(void) k; dag.forEach([&](auto &k) {
++nVisited; (void)k;
}); ++nVisited;
REQUIRE(nVisited == dag.size()); });
} REQUIRE(nVisited == dag.size());
}
} }

60
tests/unit_dagrun_loggers.cpp
View File

@@ -1,8 +1,7 @@
#include <iostream> #include <catch2/catch.hpp>
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <iostream>
#include <catch2/catch.hpp>
#include "daggy/loggers/dag_run/FileSystemLogger.hpp" #include "daggy/loggers/dag_run/FileSystemLogger.hpp"
#include "daggy/loggers/dag_run/OStreamLogger.hpp" #include "daggy/loggers/dag_run/OStreamLogger.hpp"
@@ -13,25 +12,32 @@ using namespace daggy;
using namespace daggy::loggers::dag_run; using namespace daggy::loggers::dag_run;
const TaskSet SAMPLE_TASKS{ const TaskSet SAMPLE_TASKS{
{"work_a", Task{.job{{"command", std::vector<std::string>{"/bin/echo", "a"}}}, .children{"c"}}}, {"work_a",
{"work_b", Task{.job{{"command", std::vector<std::string>{"/bin/echo", "b"}}}, .children{"c"}}}, Task{.job{{"command", std::vector<std::string>{"/bin/echo", "a"}}},
{"work_c", Task{.job{{"command", std::vector<std::string>{"/bin/echo", "c"}}}}} .children{"c"}}},
}; {"work_b",
Task{.job{{"command", std::vector<std::string>{"/bin/echo", "b"}}},
.children{"c"}}},
{"work_c",
Task{.job{{"command", std::vector<std::string>{"/bin/echo", "c"}}}}}};
inline DAGRunID testDAGRunInit(DAGRunLogger &logger, const std::string &name, const TaskSet &tasks) { inline DAGRunID testDAGRunInit(DAGRunLogger &logger, const std::string &name,
auto runID = logger.startDAGRun(name, tasks); const TaskSet &tasks)
auto dagRun = logger.getDAGRun(runID); {
auto runID = logger.startDAGRun(name, tasks);
auto dagRun = logger.getDAGRun(runID);
REQUIRE(dagRun.tasks == tasks); REQUIRE(dagRun.tasks == tasks);
REQUIRE(dagRun.taskRunStates.size() == tasks.size()); REQUIRE(dagRun.taskRunStates.size() == tasks.size());
auto nonQueuedTask = std::find_if(dagRun.taskRunStates.begin(), dagRun.taskRunStates.end(), auto nonQueuedTask =
[](const auto &a) { return a.second != +RunState::QUEUED; }); std::find_if(dagRun.taskRunStates.begin(), dagRun.taskRunStates.end(),
REQUIRE(nonQueuedTask == dagRun.taskRunStates.end()); [](const auto &a) { return a.second != +RunState::QUEUED; });
REQUIRE(nonQueuedTask == dagRun.taskRunStates.end());
REQUIRE(dagRun.dagStateChanges.size() == 1); REQUIRE(dagRun.dagStateChanges.size() == 1);
REQUIRE(dagRun.dagStateChanges.back().newState == +RunState::QUEUED); REQUIRE(dagRun.dagStateChanges.back().newState == +RunState::QUEUED);
return runID; return runID;
} }
/* /*
@@ -54,14 +60,16 @@ TEST_CASE("Filesystem Logger", "[filesystem_logger]") {
} }
*/ */
TEST_CASE("ostream_logger", "[ostream_logger]") { TEST_CASE("ostream_logger", "[ostream_logger]")
//cleanup(); {
std::stringstream ss; // cleanup();
daggy::loggers::dag_run::OStreamLogger logger(ss); std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);
SECTION("DAGRun Starts") { SECTION("DAGRun Starts")
testDAGRunInit(logger, "init_test", SAMPLE_TASKS); {
} testDAGRunInit(logger, "init_test", SAMPLE_TASKS);
}
// cleanup(); // cleanup();
} }

153
tests/unit_executor_forkingexecutor.cpp
View File

@@ -1,86 +1,103 @@
#include <iostream> #include <catch2/catch.hpp>
#include <filesystem> #include <filesystem>
#include <iostream>
#include "daggy/executors/task/ForkingTaskExecutor.hpp"
#include "daggy/Serialization.hpp" #include "daggy/Serialization.hpp"
#include "daggy/Utilities.hpp" #include "daggy/Utilities.hpp"
#include "daggy/executors/task/ForkingTaskExecutor.hpp"
#include <catch2/catch.hpp> TEST_CASE("forking_executor", "[forking_executor]")
{
daggy::executors::task::ForkingTaskExecutor ex(10);
TEST_CASE("forking_executor", "[forking_executor]") { SECTION("Simple Run")
daggy::executors::task::ForkingTaskExecutor ex(10); {
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}};
SECTION("Simple Run") { REQUIRE(ex.validateTaskParameters(task.job));
daggy::Task task{.job{
{"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job)); auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
auto recFuture = ex.execute("command", task); REQUIRE(rec.rc == 0);
auto rec = recFuture.get(); REQUIRE(rec.outputLog.size() >= 6);
REQUIRE(rec.errorLog.empty());
}
REQUIRE(rec.rc == 0); SECTION("Error Run")
REQUIRE(rec.outputLog.size() >= 6); {
REQUIRE(rec.errorLog.empty()); daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/expr", "1", "+", "+"}}}};
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc == 2);
REQUIRE(rec.errorLog.size() >= 20);
REQUIRE(rec.outputLog.empty());
}
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
if (!std::filesystem::exists(bigFile))
continue;
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));
REQUIRE(rec.errorLog.empty());
} }
}
SECTION("Error Run") { SECTION("Parameter Expansion")
daggy::Task task{.job{ {
{"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}}; std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
auto recFuture = ex.execute("command", task); std::string taskJSON =
auto rec = recFuture.get(); R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
REQUIRE(rec.rc == 2); auto result = daggy::expandTaskSet(tasks, ex, params);
REQUIRE(rec.errorLog.size() >= 20); REQUIRE(result.size() == 2);
REQUIRE(rec.outputLog.empty()); }
}
SECTION("Large Output") { SECTION("Build with expansion")
const std::vector<std::string> BIG_FILES{ {
"/usr/share/dict/linux.words", "/usr/share/dict/cracklib-small", "/etc/ssh/moduli" std::string testParams{
}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks =
R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks =
daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params);
REQUIRE(tasks.size() == 4);
}
for (const auto &bigFile: BIG_FILES) { SECTION("Build with expansion using parents instead of children")
if (!std::filesystem::exists(bigFile)) continue; {
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks =
R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})";
auto tasks =
daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params);
daggy::Task task{.job{ REQUIRE(tasks.size() == 4);
{"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/cat", bigFile}}}}; }
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));
REQUIRE(rec.errorLog.empty());
}
}
SECTION("Parameter Expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string taskJSON = R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params);
REQUIRE(result.size() == 2);
}
SECTION("Build with expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params);
REQUIRE(tasks.size() == 4);
}
SECTION("Build with expansion using parents instead of children") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params);
REQUIRE(tasks.size() == 4);
}
} }

189
tests/unit_executor_slurmexecutor.cpp
View File

@@ -1,111 +1,124 @@
#include <iostream>
#include <filesystem>
#include <unistd.h>
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h>
#include "daggy/executors/task/SlurmTaskExecutor.hpp"
#include "daggy/Serialization.hpp"
#include "daggy/Utilities.hpp"
#include <catch2/catch.hpp> #include <catch2/catch.hpp>
#include <filesystem>
#include <iostream>
#include "daggy/Serialization.hpp"
#include "daggy/Utilities.hpp"
#include "daggy/executors/task/SlurmTaskExecutor.hpp"
namespace fs = std::filesystem; namespace fs = std::filesystem;
#ifdef DAGGY_ENABLE_SLURM #ifdef DAGGY_ENABLE_SLURM
TEST_CASE("slurm_execution", "[slurm_executor]") { TEST_CASE("slurm_execution", "[slurm_executor]")
daggy::executors::task::SlurmTaskExecutor ex; {
daggy::executors::task::SlurmTaskExecutor ex;
daggy::ConfigValues defaultJobValues{ daggy::ConfigValues defaultJobValues{{"minCPUs", "1"},
{"minCPUs", "1"}, {"minMemoryMB", "100"},
{"minMemoryMB", "100"}, {"minTmpDiskMB", "0"},
{"minTmpDiskMB", "0"}, {"priority", "1"},
{"priority", "1"}, {"timeLimitSeconds", "200"},
{"timeLimitSeconds", "200"}, {"userID", std::to_string(getuid())},
{"userID", std::to_string(getuid())}, {"workDir", fs::current_path().string()},
{"workDir", fs::current_path().string()}, {"tmpDir", fs::current_path().string()}};
{"tmpDir", fs::current_path().string()}
};
SECTION("Simple Run") { SECTION("Simple Run")
daggy::Task task{.job{ {
{"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}} daggy::Task task{.job{
}}; {"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
REQUIRE(ex.validateTaskParameters(task.job)); REQUIRE(ex.validateTaskParameters(task.job));
auto recFuture = ex.execute("command", task); auto recFuture = ex.execute("command", task);
auto rec = recFuture.get(); auto rec = recFuture.get();
REQUIRE(rec.rc == 0); REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() >= 6); REQUIRE(rec.outputLog.size() >= 6);
REQUIRE(rec.errorLog.empty()); REQUIRE(rec.errorLog.empty());
}
SECTION("Error Run")
{
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/expr", "1", "+", "+"}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc != 0);
REQUIRE(rec.errorLog.size() >= 20);
REQUIRE(rec.outputLog.empty());
}
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
if (!std::filesystem::exists(bigFile))
continue;
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));
REQUIRE(rec.errorLog.empty());
break;
} }
}
SECTION("Error Run") { SECTION("Parameter Expansion")
daggy::Task task{.job{ {
{"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}}; std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
task.job.merge(defaultJobValues); auto params = daggy::configFromJSON(testParams);
auto recFuture = ex.execute("command", task); std::string taskJSON =
auto rec = recFuture.get(); R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
REQUIRE(rec.rc != 0); auto result = daggy::expandTaskSet(tasks, ex, params);
REQUIRE(rec.errorLog.size() >= 20); REQUIRE(result.size() == 2);
REQUIRE(rec.outputLog.empty()); }
}
SECTION("Large Output") { SECTION("Build with expansion")
const std::vector<std::string> BIG_FILES{ {
"/usr/share/dict/linux.words", "/usr/share/dict/cracklib-small", "/etc/ssh/moduli" std::string testParams{
}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks =
R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::expandTaskSet(
daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
REQUIRE(tasks.size() == 4);
}
for (const auto &bigFile: BIG_FILES) { SECTION("Build with expansion using parents instead of children")
if (!std::filesystem::exists(bigFile)) continue; {
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks =
R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})";
auto tasks = daggy::expandTaskSet(
daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
daggy::Task task{.job{ REQUIRE(tasks.size() == 4);
{"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/cat", bigFile}}}}; }
task.job.merge(defaultJobValues);
auto recFuture = ex.execute("command", task);
auto rec = recFuture.get();
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));
REQUIRE(rec.errorLog.empty());
break;
}
}
SECTION("Parameter Expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string taskJSON = R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
auto result = daggy::expandTaskSet(tasks, ex, params);
REQUIRE(result.size() == 2);
}
SECTION("Build with expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
REQUIRE(tasks.size() == 4);
}
SECTION("Build with expansion using parents instead of children") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
REQUIRE(tasks.size() == 4);
}
} }
#endif #endif

112
tests/unit_serialization.cpp
View File

@@ -1,35 +1,48 @@
#include <iostream> #include <catch2/catch.hpp>
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <iostream>
#include <catch2/catch.hpp>
#include "daggy/Serialization.hpp" #include "daggy/Serialization.hpp"
namespace fs = std::filesystem; namespace fs = std::filesystem;
TEST_CASE("parameter_deserialization", "[deserialize_parameters]") { TEST_CASE("parameter_deserialization", "[deserialize_parameters]")
SECTION("Basic Parse") { {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; SECTION("Basic Parse")
auto params = daggy::configFromJSON(testParams); {
REQUIRE(params.size() == 2); std::string testParams{
REQUIRE(std::holds_alternative<std::vector<std::string>>(params["DATE"])); R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
REQUIRE(std::holds_alternative<std::string>(params["SOURCE"])); auto params = daggy::configFromJSON(testParams);
}SECTION("Invalid JSON") { REQUIRE(params.size() == 2);
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name")"}; REQUIRE(std::holds_alternative<std::vector<std::string>>(params["DATE"]));
REQUIRE_THROWS(daggy::configFromJSON(testParams)); REQUIRE(std::holds_alternative<std::string>(params["SOURCE"]));
}SECTION("Non-string Keys") { }
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], 6: "name"})"}; SECTION("Invalid JSON")
REQUIRE_THROWS(daggy::configFromJSON(testParams)); {
}SECTION("Non-array/Non-string values") { std::string testParams{
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": {"name": "kevin"}})"}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name")"};
REQUIRE_THROWS(daggy::configFromJSON(testParams)); REQUIRE_THROWS(daggy::configFromJSON(testParams));
} }
SECTION("Non-string Keys")
{
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], 6: "name"})"};
REQUIRE_THROWS(daggy::configFromJSON(testParams));
}
SECTION("Non-array/Non-string values")
{
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": {"name": "kevin"}})"};
REQUIRE_THROWS(daggy::configFromJSON(testParams));
}
} }
TEST_CASE("task_deserialization", "[deserialize_task]") { TEST_CASE("task_deserialization", "[deserialize_task]")
SECTION("Build with no expansion") { {
std::string testTasks = R"({ SECTION("Build with no expansion")
{
std::string testTasks = R"({
"A": { "A": {
"job": { "command": ["/bin/echo", "A"] }, "job": { "command": ["/bin/echo", "A"] },
"children": ["C"] "children": ["C"]
@@ -42,12 +55,13 @@ TEST_CASE("task_deserialization", "[deserialize_task]") {
"job": {"command": ["/bin/echo", "C"]} "job": {"command": ["/bin/echo", "C"]}
} }
})"; })";
auto tasks = daggy::tasksFromJSON(testTasks); auto tasks = daggy::tasksFromJSON(testTasks);
REQUIRE(tasks.size() == 3); REQUIRE(tasks.size() == 3);
} }
SECTION("Build with job defaults") { SECTION("Build with job defaults")
std::string testTasks = R"({ {
std::string testTasks = R"({
"A": { "A": {
"job": { "command": ["/bin/echo", "A"] }, "job": { "command": ["/bin/echo", "A"] },
"children": ["B"] "children": ["B"]
@@ -59,30 +73,32 @@ TEST_CASE("task_deserialization", "[deserialize_task]") {
} }
} }
})"; })";
daggy::ConfigValues jobDefaults{{"runtime", "60"}, daggy::ConfigValues jobDefaults{{"runtime", "60"}, {"memory", "300M"}};
{"memory", "300M"}}; auto tasks = daggy::tasksFromJSON(testTasks, jobDefaults);
auto tasks = daggy::tasksFromJSON(testTasks, jobDefaults); REQUIRE(tasks.size() == 2);
REQUIRE(tasks.size() == 2); REQUIRE(std::get<std::string>(tasks["A"].job["runtime"]) == "60");
REQUIRE(std::get<std::string>(tasks["A"].job["runtime"]) == "60"); REQUIRE(std::get<std::string>(tasks["A"].job["memory"]) == "300M");
REQUIRE(std::get<std::string>(tasks["A"].job["memory"]) == "300M"); REQUIRE(std::get<std::string>(tasks["B"].job["runtime"]) == "60");
REQUIRE(std::get<std::string>(tasks["B"].job["runtime"]) == "60"); REQUIRE(std::get<std::string>(tasks["B"].job["memory"]) == "1G");
REQUIRE(std::get<std::string>(tasks["B"].job["memory"]) == "1G"); }
}
} }
TEST_CASE("task_serialization", "[serialize_tasks]") { TEST_CASE("task_serialization", "[serialize_tasks]")
SECTION("Build with no expansion") { {
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/echo", "B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; SECTION("Build with no expansion")
auto tasks = daggy::tasksFromJSON(testTasks); {
std::string testTasks =
R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/echo", "B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::tasksFromJSON(testTasks);
auto genJSON = daggy::tasksToJSON(tasks); auto genJSON = daggy::tasksToJSON(tasks);
auto regenTasks = daggy::tasksFromJSON(genJSON); auto regenTasks = daggy::tasksFromJSON(genJSON);
REQUIRE(regenTasks.size() == tasks.size()); REQUIRE(regenTasks.size() == tasks.size());
for (const auto &[name, task]: regenTasks) { for (const auto &[name, task] : regenTasks) {
const auto &other = tasks[name]; const auto &other = tasks[name];
REQUIRE(task == other); REQUIRE(task == other);
}
} }
}
} }

286
tests/unit_server.cpp
View File

@@ -1,83 +1,85 @@
#include <iostream>
#include <filesystem>
#include <fstream>
#include <catch2/catch.hpp>
#include <pistache/client.h> #include <pistache/client.h>
#include <rapidjson/document.h> #include <rapidjson/document.h>
#include <daggy/Server.hpp> #include <catch2/catch.hpp>
#include <daggy/Serialization.hpp> #include <daggy/Serialization.hpp>
#include <daggy/Server.hpp>
#include <daggy/executors/task/ForkingTaskExecutor.hpp> #include <daggy/executors/task/ForkingTaskExecutor.hpp>
#include <daggy/loggers/dag_run/OStreamLogger.hpp> #include <daggy/loggers/dag_run/OStreamLogger.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
namespace rj = rapidjson; namespace rj = rapidjson;
Pistache::Http::Response Pistache::Http::Response REQUEST(std::string url, std::string payload = "")
REQUEST(std::string url, std::string payload = "") { {
Pistache::Http::Experimental::Client client; Pistache::Http::Experimental::Client client;
client.init(); client.init();
Pistache::Http::Response response; Pistache::Http::Response response;
auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); auto reqSpec = (payload.empty() ? client.get(url) : client.post(url));
reqSpec.timeout(std::chrono::seconds(2)); reqSpec.timeout(std::chrono::seconds(2));
if (!payload.empty()) { if (!payload.empty()) {
reqSpec.body(payload); reqSpec.body(payload);
} }
auto request = reqSpec.send(); auto request = reqSpec.send();
bool ok = false, error = false; bool ok = false, error = false;
std::string msg; std::string msg;
request.then( request.then(
[&](Pistache::Http::Response rsp) { [&](Pistache::Http::Response rsp) {
ok = true; ok = true;
response = rsp; response = rsp;
}, },
[&](std::exception_ptr ptr) { [&](std::exception_ptr ptr) {
error = true; error = true;
try { try {
std::rethrow_exception(ptr); std::rethrow_exception(ptr);
} catch (std::exception &e) { }
msg = e.what(); catch (std::exception &e) {
} msg = e.what();
} }
); });
Pistache::Async::Barrier<Pistache::Http::Response> barrier(request); Pistache::Async::Barrier<Pistache::Http::Response> barrier(request);
barrier.wait_for(std::chrono::seconds(2)); barrier.wait_for(std::chrono::seconds(2));
client.shutdown(); client.shutdown();
if (error) { if (error) {
throw std::runtime_error(msg); throw std::runtime_error(msg);
} }
return response; return response;
} }
TEST_CASE("rest_endpoint", "[server_basic]") { TEST_CASE("rest_endpoint", "[server_basic]")
std::stringstream ss; {
daggy::executors::task::ForkingTaskExecutor executor(10); std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss); daggy::executors::task::ForkingTaskExecutor executor(10);
Pistache::Address listenSpec("localhost", Pistache::Port(0)); daggy::loggers::dag_run::OStreamLogger logger(ss);
Pistache::Address listenSpec("localhost", Pistache::Port(0));
const size_t nDAGRunners = 10, const size_t nDAGRunners = 10, nWebThreads = 10;
nWebThreads = 10;
daggy::Server server(listenSpec, logger, executor, nDAGRunners); daggy::Server server(listenSpec, logger, executor, nDAGRunners);
server.init(nWebThreads); server.init(nWebThreads);
server.start(); server.start();
const std::string host = "localhost:"; const std::string host = "localhost:";
const std::string baseURL = host + std::to_string(server.getPort()); const std::string baseURL = host + std::to_string(server.getPort());
SECTION ("Ready Endpoint") { SECTION("Ready Endpoint")
auto response = REQUEST(baseURL + "/ready"); {
REQUIRE(response.code() == Pistache::Http::Code::Ok); auto response = REQUEST(baseURL + "/ready");
} REQUIRE(response.code() == Pistache::Http::Code::Ok);
}
SECTION ("Querying a non-existent dagrunid should fail ") { SECTION("Querying a non-existent dagrunid should fail ")
auto response = REQUEST(baseURL + "/v1/dagrun/100"); {
REQUIRE(response.code() != Pistache::Http::Code::Ok); auto response = REQUEST(baseURL + "/v1/dagrun/100");
} REQUIRE(response.code() != Pistache::Http::Code::Ok);
}
SECTION("Simple DAGRun Submission") { SECTION("Simple DAGRun Submission")
std::string dagRun = R"({ {
std::string dagRun = R"({
"name": "unit_server", "name": "unit_server",
"parameters": { "FILE": [ "A", "B" ] }, "parameters": { "FILE": [ "A", "B" ] },
"tasks": { "tasks": {
@@ -88,87 +90,89 @@ TEST_CASE("rest_endpoint", "[server_basic]") {
} }
})"; })";
// Submit, and get the runID
daggy::DAGRunID runID = 0;
{
auto response = REQUEST(baseURL + "/v1/dagrun/", dagRun);
REQUIRE(response.code() == Pistache::Http::Code::Ok);
// Submit, and get the runID rj::Document doc;
daggy::DAGRunID runID = 0; daggy::checkRJParse(doc.Parse(response.body().c_str()));
{ REQUIRE(doc.IsObject());
auto response = REQUEST(baseURL + "/v1/dagrun/", dagRun); REQUIRE(doc.HasMember("runID"));
REQUIRE(response.code() == Pistache::Http::Code::Ok);
rj::Document doc; runID = doc["runID"].GetUint64();
daggy::checkRJParse(doc.Parse(response.body().c_str()));
REQUIRE(doc.IsObject());
REQUIRE(doc.HasMember("runID"));
runID = doc["runID"].GetUint64();
}
// Ensure our runID shows up in the list of running DAGs
{
auto response = REQUEST(baseURL + "/v1/dagrun/");
REQUIRE(response.code() == Pistache::Http::Code::Ok);
rj::Document doc;
daggy::checkRJParse(doc.Parse(response.body().c_str()));
REQUIRE(doc.IsArray());
REQUIRE(doc.Size() >= 1);
// Ensure that our DAG is in the list and matches our given DAGRunID
bool found = false;
const auto &runs = doc.GetArray();
for (size_t i = 0; i < runs.Size(); ++i) {
const auto &run = runs[i];
REQUIRE(run.IsObject());
REQUIRE(run.HasMember("name"));
REQUIRE(run.HasMember("runID"));
std::string runName = run["name"].GetString();
if (runName == "unit_server") {
REQUIRE(run["runID"].GetUint64() == runID);
found = true;
break;
}
}
REQUIRE(found);
}
// Wait until our DAG is complete
bool complete = true;
for (auto i = 0; i < 10; ++i) {
auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID));
REQUIRE(response.code() == Pistache::Http::Code::Ok);
rj::Document doc;
daggy::checkRJParse(doc.Parse(response.body().c_str()));
REQUIRE(doc.IsObject());
REQUIRE(doc.HasMember("taskStates"));
const auto &taskStates = doc["taskStates"].GetObject();
size_t nStates = 0;
for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); ++it) {
nStates++;
}
REQUIRE(nStates == 3);
complete = true;
for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); ++it) {
std::string state = it->value.GetString();
if (state != "COMPLETED") {
complete = false;
break;
}
}
if (complete) break;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
REQUIRE(complete);
std::this_thread::sleep_for(std::chrono::seconds(2));
for (const auto &pth: std::vector<fs::path>{"dagrun_A", "dagrun_B"}) {
REQUIRE(fs::exists(pth));
fs::remove(pth);
}
} }
server.shutdown(); // Ensure our runID shows up in the list of running DAGs
{
auto response = REQUEST(baseURL + "/v1/dagrun/");
REQUIRE(response.code() == Pistache::Http::Code::Ok);
rj::Document doc;
daggy::checkRJParse(doc.Parse(response.body().c_str()));
REQUIRE(doc.IsArray());
REQUIRE(doc.Size() >= 1);
// Ensure that our DAG is in the list and matches our given DAGRunID
bool found = false;
const auto &runs = doc.GetArray();
for (size_t i = 0; i < runs.Size(); ++i) {
const auto &run = runs[i];
REQUIRE(run.IsObject());
REQUIRE(run.HasMember("name"));
REQUIRE(run.HasMember("runID"));
std::string runName = run["name"].GetString();
if (runName == "unit_server") {
REQUIRE(run["runID"].GetUint64() == runID);
found = true;
break;
}
}
REQUIRE(found);
}
// Wait until our DAG is complete
bool complete = true;
for (auto i = 0; i < 10; ++i) {
auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID));
REQUIRE(response.code() == Pistache::Http::Code::Ok);
rj::Document doc;
daggy::checkRJParse(doc.Parse(response.body().c_str()));
REQUIRE(doc.IsObject());
REQUIRE(doc.HasMember("taskStates"));
const auto &taskStates = doc["taskStates"].GetObject();
size_t nStates = 0;
for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd();
++it) {
nStates++;
}
REQUIRE(nStates == 3);
complete = true;
for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd();
++it) {
std::string state = it->value.GetString();
if (state != "COMPLETED") {
complete = false;
break;
}
}
if (complete)
break;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
REQUIRE(complete);
std::this_thread::sleep_for(std::chrono::seconds(2));
for (const auto &pth : std::vector<fs::path>{"dagrun_A", "dagrun_B"}) {
REQUIRE(fs::exists(pth));
fs::remove(pth);
}
}
server.shutdown();
} }

66
tests/unit_threadpool.cpp
View File

@@ -1,41 +1,45 @@
#include <iostream> #include <catch2/catch.hpp>
#include <future> #include <future>
#include <iostream>
#include "daggy/ThreadPool.hpp" #include "daggy/ThreadPool.hpp"
#include <catch2/catch.hpp>
using namespace daggy; using namespace daggy;
TEST_CASE("threadpool", "[threadpool]") { TEST_CASE("threadpool", "[threadpool]")
std::atomic<uint32_t> cnt(0); {
ThreadPool tp(10); std::atomic<uint32_t> cnt(0);
ThreadPool tp(10);
std::vector<std::future<uint32_t>> rets; std::vector<std::future<uint32_t>> rets;
SECTION("Adding large tasks queues with return values") { SECTION("Adding large tasks queues with return values")
auto tq = std::make_shared<daggy::TaskQueue>(); {
std::vector<std::future<uint32_t>> res; auto tq = std::make_shared<daggy::TaskQueue>();
for (size_t i = 0; i < 100; ++i) std::vector<std::future<uint32_t>> res;
res.emplace_back(std::move(tq->addTask([&cnt]() { for (size_t i = 0; i < 100; ++i)
cnt++; res.emplace_back(std::move(tq->addTask([&cnt]() {
return cnt.load(); cnt++;
}))); return cnt.load();
tp.addTasks(tq); })));
for (auto &r: res) r.get(); tp.addTasks(tq);
REQUIRE(cnt == 100); for (auto &r : res)
} r.get();
REQUIRE(cnt == 100);
}
SECTION("Slow runs") { SECTION("Slow runs")
std::vector<std::future<void>> res; {
using namespace std::chrono_literals; std::vector<std::future<void>> res;
for (size_t i = 0; i < 100; ++i) using namespace std::chrono_literals;
res.push_back(tp.addTask([&cnt]() { for (size_t i = 0; i < 100; ++i)
std::this_thread::sleep_for(20ms); res.push_back(tp.addTask([&cnt]() {
cnt++; std::this_thread::sleep_for(20ms);
return; cnt++;
})); return;
for (auto &r: res) r.get(); }));
REQUIRE(cnt == 100); for (auto &r : res)
} r.get();
REQUIRE(cnt == 100);
}
} }

420
tests/unit_utilities.cpp
View File

@@ -1,74 +1,79 @@
#include <iostream> #include <algorithm>
#include <catch2/catch.hpp>
#include <chrono> #include <chrono>
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <iomanip> #include <iomanip>
#include <algorithm>
#include <iostream> #include <iostream>
#include <random> #include <random>
#include <catch2/catch.hpp>
#include "daggy/Utilities.hpp"
#include "daggy/Serialization.hpp" #include "daggy/Serialization.hpp"
#include "daggy/Utilities.hpp"
#include "daggy/executors/task/ForkingTaskExecutor.hpp" #include "daggy/executors/task/ForkingTaskExecutor.hpp"
#include "daggy/executors/task/NoopTaskExecutor.hpp" #include "daggy/executors/task/NoopTaskExecutor.hpp"
#include "daggy/loggers/dag_run/OStreamLogger.hpp" #include "daggy/loggers/dag_run/OStreamLogger.hpp"
namespace fs = std::filesystem; namespace fs = std::filesystem;
TEST_CASE("string_utilities", "[utilities_string]") { TEST_CASE("string_utilities", "[utilities_string]")
std::string test = "/this/is/{{A}}/test/{{A}}"; {
auto res = daggy::globalSub(test, "{{A}}", "hello"); std::string test = "/this/is/{{A}}/test/{{A}}";
REQUIRE(res == "/this/is/hello/test/hello"); auto res = daggy::globalSub(test, "{{A}}", "hello");
REQUIRE(res == "/this/is/hello/test/hello");
} }
TEST_CASE("string_expansion", "[utilities_parameter_expansion]") { TEST_CASE("string_expansion", "[utilities_parameter_expansion]")
SECTION("Basic expansion") { {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; SECTION("Basic expansion")
auto params = daggy::configFromJSON(testParams); {
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}", "{{TYPE}}"}; std::string testParams{
auto allCommands = daggy::interpolateValues(cmd, params); R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
REQUIRE(allCommands.size() == 6);
}
SECTION("Skip over unused parameters") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
REQUIRE(allCommands.size() == 2);
}
SECTION("Expand within a command part") {
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"};
auto result = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
REQUIRE(result.size() == 4);
}
}
TEST_CASE("dag_runner_order", "[dagrun_order]") {
daggy::executors::task::NoopTaskExecutor ex;
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);
daggy::TimePoint startTime = daggy::Clock::now();
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07", "2021-05-08", "2021-05-09" ]})"};
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}",
"{{TYPE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
std::string taskJSON = R"({ REQUIRE(allCommands.size() == 6);
}
SECTION("Skip over unused parameters")
{
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
REQUIRE(allCommands.size() == 2);
}
SECTION("Expand within a command part")
{
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"};
auto result = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
REQUIRE(result.size() == 4);
}
}
TEST_CASE("dag_runner_order", "[dagrun_order]")
{
daggy::executors::task::NoopTaskExecutor ex;
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);
daggy::TimePoint startTime = daggy::Clock::now();
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07", "2021-05-08", "2021-05-09" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string taskJSON = R"({
"A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]}, "A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]}, "B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]}, "C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
@@ -76,160 +81,175 @@ TEST_CASE("dag_runner_order", "[dagrun_order]") {
"E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}} "E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}}
})"; })";
auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex, params); auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex, params);
REQUIRE(tasks.size() == 20); REQUIRE(tasks.size() == 20);
auto dag = daggy::buildDAGFromTasks(tasks); auto dag = daggy::buildDAGFromTasks(tasks);
auto runID = logger.startDAGRun("test_run", tasks); auto runID = logger.startDAGRun("test_run", tasks);
auto endDAG = daggy::runDAG(runID, ex, logger, dag);
REQUIRE(endDAG.allVisited());
// Ensure the run order
auto rec = logger.getDAGRun(runID);
daggy::TimePoint stopTime = daggy::Clock::now();
std::array<daggy::TimePoint, 5> minTimes;
minTimes.fill(startTime);
std::array<daggy::TimePoint, 5> maxTimes;
maxTimes.fill(stopTime);
for (const auto &[k, v] : rec.taskAttempts) {
size_t idx = k[0] - 65;
auto &startTime = minTimes[idx];
auto &stopTime = maxTimes[idx];
startTime = std::max(startTime, v.front().startTime);
stopTime = std::min(stopTime, v.back().stopTime);
}
for (size_t i = 0; i < 5; ++i) {
for (size_t j = i + 1; j < 4; ++j) {
REQUIRE(maxTimes[i] < minTimes[j]);
}
}
}
TEST_CASE("dag_runner", "[utilities_dag_runner]")
{
daggy::executors::task::ForkingTaskExecutor ex(10);
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);
SECTION("Simple execution")
{
std::string prefix = (fs::current_path() / "asdlk").string();
std::unordered_map<std::string, std::string> files{
{"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}};
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
files.at("B") +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
files.at("C") + R"("]}}})";
auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(tasks);
auto runID = logger.startDAGRun("test_run", tasks);
auto endDAG = daggy::runDAG(runID, ex, logger, dag); auto endDAG = daggy::runDAG(runID, ex, logger, dag);
REQUIRE(endDAG.allVisited()); REQUIRE(endDAG.allVisited());
// Ensure the run order for (const auto &[_, file] : files) {
auto rec = logger.getDAGRun(runID); REQUIRE(fs::exists(file));
fs::remove(file);
daggy::TimePoint stopTime = daggy::Clock::now();
std::array<daggy::TimePoint, 5> minTimes; minTimes.fill(startTime);
std::array<daggy::TimePoint, 5> maxTimes; maxTimes.fill(stopTime);
for (const auto &[k, v] : rec.taskAttempts) {
size_t idx = k[0] - 65;
auto & startTime = minTimes[idx];
auto & stopTime = maxTimes[idx];
startTime = std::max(startTime, v.front().startTime);
stopTime = std::min(stopTime, v.back().stopTime);
} }
for (size_t i = 0; i < 5; ++i) { // Get the DAG Run Attempts
for (size_t j = i+1; j < 4; ++j) { auto record = logger.getDAGRun(runID);
REQUIRE(maxTimes[i] < minTimes[j]); for (const auto &[_, attempts] : record.taskAttempts) {
} REQUIRE(attempts.size() == 1);
} REQUIRE(attempts.front().rc == 0);
}
TEST_CASE("dag_runner", "[utilities_dag_runner]") {
daggy::executors::task::ForkingTaskExecutor ex(10);
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);
SECTION("Simple execution") {
std::string prefix = (fs::current_path() / "asdlk").string();
std::unordered_map<std::string, std::string> files{
{"A", prefix + "_A"},
{"B", prefix + "_B"},
{"C", prefix + "_C"}};
std::string taskJSON = R"({"A": {"job": {"command": ["/usr/bin/touch", ")"
+ files.at("A") + R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")"
+ files.at("B") + R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")"
+ files.at("C") + R"("]}}})";
auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(tasks);
auto runID = logger.startDAGRun("test_run", tasks);
auto endDAG = daggy::runDAG(runID, ex, logger, dag);
REQUIRE(endDAG.allVisited());
for (const auto &[_, file] : files) {
REQUIRE(fs::exists(file));
fs::remove(file);
}
// Get the DAG Run Attempts
auto record = logger.getDAGRun(runID);
for (const auto &[_, attempts]: record.taskAttempts) {
REQUIRE(attempts.size() == 1);
REQUIRE(attempts.front().rc == 0);
}
}
SECTION("Recovery from Error") {
auto cleanup = []() {
// Cleanup
std::vector<fs::path> paths{"rec_error_A", "noexist"};
for (const auto &pth: paths) {
if (fs::exists(pth)) fs::remove_all(pth);
}
};
cleanup();
std::string goodPrefix = "rec_error_";
std::string badPrefix = "noexist/rec_error_";
std::string taskJSON = R"({"A": {"job": {"command": ["/usr/bin/touch", ")"
+ goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")"
+ badPrefix +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")"
+ badPrefix + R"(C"]}}})";
auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(tasks);
auto runID = logger.startDAGRun("test_run", tasks);
auto tryDAG = daggy::runDAG(runID, ex, logger, dag);
REQUIRE(!tryDAG.allVisited());
// Create the missing dir, then continue to run the DAG
fs::create_directory("noexist");
tryDAG.resetRunning();
auto endDAG = daggy::runDAG(runID, ex, logger, tryDAG);
REQUIRE(endDAG.allVisited());
// Get the DAG Run Attempts
auto record = logger.getDAGRun(runID);
REQUIRE(record.taskAttempts["A_0"].size() == 1); // A ran fine
REQUIRE(record.taskAttempts["B_0"].size() == 2); // B errored and had to be retried
REQUIRE(record.taskAttempts["C_0"].size() == 1); // C wasn't run because B errored
cleanup();
}
SECTION("Generator tasks") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string generatorOutput = R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
fs::path ofn = fs::current_path() / "generator_test_output.json";
std::ofstream ofh{ofn};
ofh << generatorOutput << std::endl;
ofh.close();
std::stringstream jsonTasks;
jsonTasks << R"({ "A": { "job": {"command": [ "/usr/bin/cat", )" << std::quoted(ofn.string())
<< R"(]}, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
auto baseTasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(baseTasks.size() == 2);
auto tasks = daggy::expandTaskSet(baseTasks, ex, params);
REQUIRE(tasks.size() == 2);
auto dag = daggy::buildDAGFromTasks(tasks);
REQUIRE(dag.size() == 2);
auto runID = logger.startDAGRun("generator_run", tasks);
auto finalDAG = daggy::runDAG(runID, ex, logger, dag, params);
REQUIRE(finalDAG.allVisited());
REQUIRE(finalDAG.size() == 4);
// Check the logger
auto record = logger.getDAGRun(runID);
REQUIRE(record.tasks.size() == 4);
REQUIRE(record.taskRunStates.size() == 4);
for (const auto &[taskName, attempts]: record.taskAttempts) {
REQUIRE(attempts.size() == 1);
REQUIRE(attempts.back().rc == 0);
}
// Ensure that children were updated properly
REQUIRE(record.tasks["A_0"].children == std::unordered_set<std::string>{"B_0", "B_1", "C"});
REQUIRE(record.tasks["B_0"].children == std::unordered_set<std::string>{"C"});
REQUIRE(record.tasks["B_1"].children == std::unordered_set<std::string>{"C"});
REQUIRE(record.tasks["C_0"].children.empty());
} }
}
SECTION("Recovery from Error")
{
auto cleanup = []() {
// Cleanup
std::vector<fs::path> paths{"rec_error_A", "noexist"};
for (const auto &pth : paths) {
if (fs::exists(pth))
fs::remove_all(pth);
}
};
cleanup();
std::string goodPrefix = "rec_error_";
std::string badPrefix = "noexist/rec_error_";
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
badPrefix +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
badPrefix + R"(C"]}}})";
auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(tasks);
auto runID = logger.startDAGRun("test_run", tasks);
auto tryDAG = daggy::runDAG(runID, ex, logger, dag);
REQUIRE(!tryDAG.allVisited());
// Create the missing dir, then continue to run the DAG
fs::create_directory("noexist");
tryDAG.resetRunning();
auto endDAG = daggy::runDAG(runID, ex, logger, tryDAG);
REQUIRE(endDAG.allVisited());
// Get the DAG Run Attempts
auto record = logger.getDAGRun(runID);
REQUIRE(record.taskAttempts["A_0"].size() == 1); // A ran fine
REQUIRE(record.taskAttempts["B_0"].size() ==
2); // B errored and had to be retried
REQUIRE(record.taskAttempts["C_0"].size() ==
1); // C wasn't run because B errored
cleanup();
}
SECTION("Generator tasks")
{
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string generatorOutput =
R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
fs::path ofn = fs::current_path() / "generator_test_output.json";
std::ofstream ofh{ofn};
ofh << generatorOutput << std::endl;
ofh.close();
std::stringstream jsonTasks;
jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", )"
<< std::quoted(ofn.string())
<< R"(]}, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
auto baseTasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(baseTasks.size() == 2);
auto tasks = daggy::expandTaskSet(baseTasks, ex, params);
REQUIRE(tasks.size() == 2);
auto dag = daggy::buildDAGFromTasks(tasks);
REQUIRE(dag.size() == 2);
auto runID = logger.startDAGRun("generator_run", tasks);
auto finalDAG = daggy::runDAG(runID, ex, logger, dag, params);
REQUIRE(finalDAG.allVisited());
REQUIRE(finalDAG.size() == 4);
// Check the logger
auto record = logger.getDAGRun(runID);
REQUIRE(record.tasks.size() == 4);
REQUIRE(record.taskRunStates.size() == 4);
for (const auto &[taskName, attempts] : record.taskAttempts) {
REQUIRE(attempts.size() == 1);
REQUIRE(attempts.back().rc == 0);
}
// Ensure that children were updated properly
REQUIRE(record.tasks["A_0"].children ==
std::unordered_set<std::string>{"B_0", "B_1", "C"});
REQUIRE(record.tasks["B_0"].children ==
std::unordered_set<std::string>{"C"});
REQUIRE(record.tasks["B_1"].children ==
std::unordered_set<std::string>{"C"});
REQUIRE(record.tasks["C_0"].children.empty());
}
} }

129
utils/daggyc/daggyc.cpp
View File

@@ -1,78 +1,77 @@
#include <iostream>
#include <fstream>
#include <filesystem>
#include <argparse.hpp>
#include <pistache/client.h> #include <pistache/client.h>
#include <rapidjson/document.h> #include <rapidjson/document.h>
#include <argparse.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
namespace rj = rapidjson; namespace rj = rapidjson;
Pistache::Http::Response Pistache::Http::Response REQUEST(std::string url, std::string payload = "")
REQUEST(std::string url, std::string payload = "") { {
Pistache::Http::Experimental::Client client; Pistache::Http::Experimental::Client client;
client.init(); client.init();
Pistache::Http::Response response; Pistache::Http::Response response;
auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); auto reqSpec = (payload.empty() ? client.get(url) : client.post(url));
reqSpec.timeout(std::chrono::seconds(2)); reqSpec.timeout(std::chrono::seconds(2));
if (!payload.empty()) { if (!payload.empty()) {
reqSpec.body(payload); reqSpec.body(payload);
} }
auto request = reqSpec.send(); auto request = reqSpec.send();
bool ok = false, error = false; bool ok = false, error = false;
std::string msg; std::string msg;
request.then( request.then(
[&](Pistache::Http::Response rsp) { [&](Pistache::Http::Response rsp) {
ok = true; ok = true;
response = rsp; response = rsp;
}, },
[&](std::exception_ptr ptr) { [&](std::exception_ptr ptr) {
error = true; error = true;
try { try {
std::rethrow_exception(ptr); std::rethrow_exception(ptr);
} catch (std::exception &e) { }
msg = e.what(); catch (std::exception &e) {
} msg = e.what();
} }
); });
Pistache::Async::Barrier<Pistache::Http::Response> barrier(request); Pistache::Async::Barrier<Pistache::Http::Response> barrier(request);
barrier.wait_for(std::chrono::seconds(2)); barrier.wait_for(std::chrono::seconds(2));
client.shutdown(); client.shutdown();
if (error) { if (error) {
throw std::runtime_error(msg); throw std::runtime_error(msg);
} }
return response; return response;
} }
int main(int argc, char **argv) { int main(int argc, char **argv)
argparse::ArgumentParser args("Daggy Client"); {
argparse::ArgumentParser args("Daggy Client");
args.add_argument("-v", "--verbose") args.add_argument("-v", "--verbose")
.default_value(false) .default_value(false)
.implicit_value(true); .implicit_value(true);
args.add_argument("--url") args.add_argument("--url")
.help("base URL of server") .help("base URL of server")
.default_value("http://localhost:2503"); .default_value("http://localhost:2503");
args.add_argument("--sync") args.add_argument("--sync").default_value(false).implicit_value(true).help(
.default_value(false) "Poll for job to complete");
.implicit_value(true) args.add_argument("--action")
.help("Poll for job to complete"); .help("Number of tasks to run concurrently")
args.add_argument("--action") .default_value(30)
.help("Number of tasks to run concurrently") .action([](const std::string &value) { return std::stoull(value); });
.default_value(30)
.action([](const std::string &value) { return std::stoull(value); });
try { try {
args.parse_args(argc, argv); args.parse_args(argc, argv);
} catch (std::exception &e) { }
std::cout << "Error: " << e.what() << std::endl; catch (std::exception &e) {
std::cout << args; std::cout << "Error: " << e.what() << std::endl;
exit(1); std::cout << args;
} exit(1);
}
std::string baseURL = args.get<std::string>("--url"); std::string baseURL = args.get<std::string>("--url");
auto response = REQUEST(baseURL + "/ready"); auto response = REQUEST(baseURL + "/ready");
} }

312
utils/daggyd/daggyd.cpp
View File

@@ -1,13 +1,11 @@
#include <iostream>
#include <fstream>
#include <atomic>
#include <sys/stat.h>
#include <signal.h> #include <signal.h>
#include <sys/stat.h>
#include <argparse.hpp> #include <argparse.hpp>
#include <atomic>
#include <daggy/Server.hpp> #include <daggy/Server.hpp>
#include <fstream>
#include <iostream>
// Add executors here // Add executors here
#ifdef DAGGY_ENABLE_SLURM #ifdef DAGGY_ENABLE_SLURM
@@ -24,182 +22,198 @@
/* /*
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h>
#include <sys/types.h> #include <sys/types.h>
#include <syslog.h> #include <syslog.h>
#include <unistd.h>
*/ */
static std::atomic<bool> running{true}; static std::atomic<bool> running{true};
void signalHandler(int signal) { void signalHandler(int signal)
switch (signal) { {
case SIGHUP: switch (signal) {
break; case SIGHUP:
case SIGINT: break;
case SIGTERM: case SIGINT:
running = false; case SIGTERM:
break; running = false;
} break;
}
} }
void daemonize() { void daemonize()
pid_t pid; {
pid_t pid;
struct sigaction newSigAction; struct sigaction newSigAction;
sigset_t newSigSet; sigset_t newSigSet;
/* Check if parent process id is set */ /* Check if parent process id is set */
if (getppid() == 1) { return; } if (getppid() == 1) {
return;
}
/* Set signal mask - signals we want to block */ /* Set signal mask - signals we want to block */
sigemptyset(&newSigSet); sigemptyset(&newSigSet);
sigaddset(&newSigSet, SIGCHLD); /* ignore child - i.e. we don't need to wait for it */ sigaddset(&newSigSet,
sigaddset(&newSigSet, SIGTSTP); /* ignore Tty stop signals */ SIGCHLD); /* ignore child - i.e. we don't need to wait for it */
sigaddset(&newSigSet, SIGTTOU); /* ignore Tty background writes */ sigaddset(&newSigSet, SIGTSTP); /* ignore Tty stop signals */
sigaddset(&newSigSet, SIGTTIN); /* ignore Tty background reads */ sigaddset(&newSigSet, SIGTTOU); /* ignore Tty background writes */
sigprocmask(SIG_BLOCK, &newSigSet, NULL); /* Block the above specified signals */ sigaddset(&newSigSet, SIGTTIN); /* ignore Tty background reads */
sigprocmask(SIG_BLOCK, &newSigSet,
NULL); /* Block the above specified signals */
/* Set up a signal handler */ /* Set up a signal handler */
newSigAction.sa_handler = signalHandler; newSigAction.sa_handler = signalHandler;
sigemptyset(&newSigAction.sa_mask); sigemptyset(&newSigAction.sa_mask);
newSigAction.sa_flags = 0; newSigAction.sa_flags = 0;
/* Signals to handle */ /* Signals to handle */
sigaction(SIGHUP, &newSigAction, NULL); /* catch hangup signal */ sigaction(SIGHUP, &newSigAction, NULL); /* catch hangup signal */
sigaction(SIGTERM, &newSigAction, NULL); /* catch term signal */ sigaction(SIGTERM, &newSigAction, NULL); /* catch term signal */
sigaction(SIGINT, &newSigAction, NULL); /* catch interrupt signal */ sigaction(SIGINT, &newSigAction, NULL); /* catch interrupt signal */
// Fork once // Fork once
pid = fork(); pid = fork();
if (pid < 0) { exit(EXIT_FAILURE); } if (pid < 0) {
if (pid > 0) { exit(EXIT_SUCCESS); } exit(EXIT_FAILURE);
}
if (pid > 0) {
exit(EXIT_SUCCESS);
}
/* On success: The child process becomes session leader */ /* On success: The child process becomes session leader */
if (setsid() < 0) { if (setsid() < 0) {
std::cerr << "Unable to setsid" << std::endl; std::cerr << "Unable to setsid" << std::endl;
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
/* Catch, ignore and handle signals */ /* Catch, ignore and handle signals */
signal(SIGCHLD, SIG_IGN); signal(SIGCHLD, SIG_IGN);
signal(SIGHUP, SIG_IGN); signal(SIGHUP, SIG_IGN);
/* Fork off for the second time*/ /* Fork off for the second time*/
pid = fork(); pid = fork();
if (pid < 0) if (pid < 0)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
if (pid > 0) if (pid > 0)
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
umask(0); umask(0);
/* Change the working directory to the root directory */ /* Change the working directory to the root directory */
/* or another appropriated directory */ /* or another appropriated directory */
auto rc = chdir("/"); auto rc = chdir("/");
(void)rc; (void)rc;
/* Close all open file descriptors */ /* Close all open file descriptors */
for (auto x = sysconf(_SC_OPEN_MAX); x >= 0; x--) { close(x); } for (auto x = sysconf(_SC_OPEN_MAX); x >= 0; x--) {
close(x);
}
} }
int main(int argc, char **argv) { int main(int argc, char **argv)
argparse::ArgumentParser args("Daggy"); {
argparse::ArgumentParser args("Daggy");
args.add_argument("-v", "--verbose") args.add_argument("-v", "--verbose")
.default_value(false) .default_value(false)
.implicit_value(true); .implicit_value(true);
args.add_argument("-d", "--daemon") args.add_argument("-d", "--daemon").default_value(false).implicit_value(true);
.default_value(false) args.add_argument("--ip")
.implicit_value(true); .help("IP address to listen to")
args.add_argument("--ip") .default_value(std::string{"127.0.0.1"});
.help("IP address to listen to") args.add_argument("--log-file")
.default_value(std::string{"127.0.0.1"}); .help("File to log to.")
args.add_argument("--log-file") .default_value(std::string{"daggyd.log"});
.help("File to log to.") args.add_argument("--port")
.default_value(std::string{"daggyd.log"}); .help("Port to listen to")
args.add_argument("--port") .default_value(2503)
.help("Port to listen to") .action([](const std::string &value) { return std::stoi(value); });
.default_value(2503) args.add_argument("--dag-threads")
.action([](const std::string &value) { return std::stoi(value); }); .help("Number of DAGs to run concurrently")
args.add_argument("--dag-threads") .default_value(10UL)
.help("Number of DAGs to run concurrently") .action([](const std::string &value) { return std::stoull(value); });
.default_value(10UL) args.add_argument("--web-threads")
.action([](const std::string &value) { return std::stoull(value); }); .help("Number of web requests to support concurrently")
args.add_argument("--web-threads") .default_value(30UL)
.help("Number of web requests to support concurrently") .action([](const std::string &value) { return std::stoull(value); });
.default_value(30UL) args.add_argument("--executor-threads")
.action([](const std::string &value) { return std::stoull(value); }); .help("Number of tasks to run concurrently")
args.add_argument("--executor-threads") .default_value(30UL)
.help("Number of tasks to run concurrently") .action([](const std::string &value) { return std::stoull(value); });
.default_value(30UL)
.action([](const std::string &value) { return std::stoull(value); });
try { try {
args.parse_args(argc, argv); args.parse_args(argc, argv);
} catch (std::exception &e) { }
std::cout << "Error: " << e.what() << std::endl; catch (std::exception &e) {
std::cout << args; std::cout << "Error: " << e.what() << std::endl;
exit(1); std::cout << args;
} exit(1);
}
bool verbose = args.get<bool>("--verbose"); bool verbose = args.get<bool>("--verbose");
bool asDaemon = args.get<bool>("--daemon"); bool asDaemon = args.get<bool>("--daemon");
std::string logFileName = args.get<std::string>("--log-file"); std::string logFileName = args.get<std::string>("--log-file");
std::string listenIP = args.get<std::string>("--ip"); std::string listenIP = args.get<std::string>("--ip");
uint16_t listenPort = args.get<int>("--port"); uint16_t listenPort = args.get<int>("--port");
size_t executorThreads = args.get<size_t>("--executor-threads"); size_t executorThreads = args.get<size_t>("--executor-threads");
size_t webThreads = args.get<size_t>("--web-threads"); size_t webThreads = args.get<size_t>("--web-threads");
size_t dagThreads = args.get<size_t>("--dag-threads"); size_t dagThreads = args.get<size_t>("--dag-threads");
if (logFileName == "-") {
if (asDaemon) {
std::cout << "Unable to daemonize if logging to stdout" << std::endl;
exit(1);
}
} else {
fs::path logFn{logFileName};
if (!logFn.is_absolute()) {
logFileName = (fs::current_path() / logFileName).string();
}
}
if (verbose) {
std::cout << "Server running at http://" << listenIP << ':' << listenPort << std::endl
<< "Max DAG Processing: " << dagThreads << std::endl
<< "Max Task Execution: " << executorThreads << std::endl
<< "Max Web Clients: " << webThreads << std::endl
<< "Logging to: " << logFileName << std::endl
<< std::endl << "Ctrl-C to exit" << std::endl;
}
if (logFileName == "-") {
if (asDaemon) { if (asDaemon) {
daemonize(); std::cout << "Unable to daemonize if logging to stdout" << std::endl;
exit(1);
} }
}
else {
fs::path logFn{logFileName};
if (!logFn.is_absolute()) {
logFileName = (fs::current_path() / logFileName).string();
}
}
std::ofstream logFH; if (verbose) {
std::unique_ptr<daggy::loggers::dag_run::DAGRunLogger> logger; std::cout << "Server running at http://" << listenIP << ':' << listenPort
if (logFileName == "-") { << std::endl
logger = std::make_unique<daggy::loggers::dag_run::OStreamLogger>(std::cout); << "Max DAG Processing: " << dagThreads << std::endl
} else { << "Max Task Execution: " << executorThreads << std::endl
logFH.open(logFileName, std::ios::app); << "Max Web Clients: " << webThreads << std::endl
logger = std::make_unique<daggy::loggers::dag_run::OStreamLogger>(logFH); << "Logging to: " << logFileName << std::endl
} << std::endl
<< "Ctrl-C to exit" << std::endl;
}
if (asDaemon) {
daemonize();
}
std::ofstream logFH;
std::unique_ptr<daggy::loggers::dag_run::DAGRunLogger> logger;
if (logFileName == "-") {
logger =
std::make_unique<daggy::loggers::dag_run::OStreamLogger>(std::cout);
}
else {
logFH.open(logFileName, std::ios::app);
logger = std::make_unique<daggy::loggers::dag_run::OStreamLogger>(logFH);
}
#ifdef DAGGY_ENABLE_SLURM #ifdef DAGGY_ENABLE_SLURM
daggy::executors::task::SlurmTaskExecutor executor; daggy::executors::task::SlurmTaskExecutor executor;
#else #else
daggy::executors::task::ForkingTaskExecutor executor(executorThreads); daggy::executors::task::ForkingTaskExecutor executor(executorThreads);
#endif #endif
Pistache::Address listenSpec(listenIP, listenPort); Pistache::Address listenSpec(listenIP, listenPort);
daggy::Server server(listenSpec, *logger, executor, dagThreads); daggy::Server server(listenSpec, *logger, executor, dagThreads);
server.init(webThreads); server.init(webThreads);
server.start(); server.start();
running = true;
running = true; while (running) {
while (running) { std::this_thread::sleep_for(std::chrono::seconds(30));
std::this_thread::sleep_for(std::chrono::seconds(30)); }
} server.shutdown();
server.shutdown();
} }