From 288ce28d292ea890955f19d0821b57e2cb121dc4 Mon Sep 17 00:00:00 2001 From: Ian Roddis Date: Tue, 21 Sep 2021 09:41:11 -0300 Subject: [PATCH] Adding clang-format, and reformating all sourcecode --- .clang-format | 198 +++++++ daggy/include/daggy/DAG.hpp | 92 ++-- daggy/include/daggy/DAG.impl.hxx | 285 +++++----- daggy/include/daggy/Defines.hpp | 92 ++-- daggy/include/daggy/Serialization.hpp | 51 +- daggy/include/daggy/Server.hpp | 74 +-- daggy/include/daggy/ThreadPool.hpp | 307 ++++++----- daggy/include/daggy/Utilities.hpp | 52 +- .../executors/task/ForkingTaskExecutor.hpp | 40 +- .../daggy/executors/task/NoopTaskExecutor.hpp | 26 +- .../executors/task/SlurmTaskExecutor.hpp | 56 +- .../daggy/executors/task/TaskExecutor.hpp | 30 +- .../daggy/loggers/dag_run/DAGRunLogger.hpp | 42 +- .../include/daggy/loggers/dag_run/Defines.hpp | 62 ++- .../loggers/dag_run/FileSystemLogger.hpp | 91 ++-- .../daggy/loggers/dag_run/OStreamLogger.hpp | 63 +-- daggy/src/Serialization.cpp | 466 ++++++++-------- daggy/src/Server.cpp | 500 ++++++++++-------- daggy/src/Utilities.cpp | 411 +++++++------- .../executors/task/ForkingTaskExecutor.cpp | 202 +++---- daggy/src/executors/task/NoopTaskExecutor.cpp | 75 +-- .../src/executors/task/SlurmTaskExecutor.cpp | 492 ++++++++--------- .../src/loggers/dag_run/FileSystemLogger.cpp | 330 ++++++------ daggy/src/loggers/dag_run/OStreamLogger.cpp | 245 +++++---- tests/int_basic.cpp | 8 +- tests/main.cpp | 5 +- tests/unit_dag.cpp | 135 +++-- tests/unit_dagrun_loggers.cpp | 60 ++- tests/unit_executor_forkingexecutor.cpp | 153 +++--- tests/unit_executor_slurmexecutor.cpp | 189 ++++--- tests/unit_serialization.cpp | 112 ++-- tests/unit_server.cpp | 286 +++++----- tests/unit_threadpool.cpp | 66 +-- tests/unit_utilities.cpp | 420 ++++++++------- utils/daggyc/daggyc.cpp | 129 +++-- utils/daggyd/daggyd.cpp | 312 +++++------ 36 files changed, 3355 insertions(+), 2802 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..a0e6805 --- /dev/null +++ b/.clang-format @@ -0,0 +1,198 @@ +--- +DisableFormat: false + +Language: Cpp +Standard: Auto + +# Indentation rules +IndentWidth: 2 +AccessModifierOffset: -2 +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 4 +IndentCaseLabels: true +IndentGotoLabels: true +IndentPPDirectives: None +IndentWrappedFunctionNames: false +NamespaceIndentation: All +UseTab: Never +TabWidth: 8 + +# Brace wrapping rules +BreakBeforeBraces: Custom +BraceWrapping: + AfterEnum: true + AfterClass: true + AfterStruct: true + AfterUnion: true + + AfterNamespace: false + AfterExternBlock: false + + AfterCaseLabel: false + AfterControlStatement: false + AfterFunction: true + + BeforeCatch: true + BeforeElse: true + + IndentBraces: false + + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true + +# Line break rules +DeriveLineEnding: true +UseCRLF: false + +KeepEmptyLinesAtTheStartOfBlocks: false +MaxEmptyLinesToKeep: 1 + +BinPackArguments: true +BinPackParameters: true +ExperimentalAutoDetectBinPacking: false + +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes + +BreakInheritanceList: BeforeComma +BreakConstructorInitializers: BeforeComma +BreakBeforeInheritanceComma: true +BreakConstructorInitializersBeforeComma: true + +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakStringLiterals: true + +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true + +AllowAllConstructorInitializersOnNextLine: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false + +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: false + +# Line length rules +ColumnLimit: 80 +ReflowComments: true + +## line length penalties +## these determine where line breaks are inserted when over ColumnLimit +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 + +# Alignment rules +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +DerivePointerAlignment: true +PointerAlignment: Left + +# Include ordering rules +IncludeBlocks: Regroup +SortIncludes: true + +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' + +IncludeCategories: + - Regex: '^".*\.h"' + Priority: 2 + SortPriority: 0 + - Regex: '^<.*\.h>' + Priority: 1 + SortPriority: 0 + - Regex: '^<.*' + Priority: 2 + SortPriority: 0 + - Regex: '.*' + Priority: 3 + SortPriority: 0 + +# Namespace rules +CompactNamespaces: true +FixNamespaceComments: true + +# Language extention macros +CommentPragmas: '^ IWYU pragma:' +MacroBlockBegin: '' +MacroBlockEnd: '' +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION + +# Spacing rules +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInParentheses: false +SpacesInSquareBrackets: false + +# Rules for detecting embedded code blocks +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google + +# C++ specific rules +Cpp11BracedListStyle: true +SortUsingDeclarations: true +... diff --git a/daggy/include/daggy/DAG.hpp b/daggy/include/daggy/DAG.hpp index b43886b..53645f8 100644 --- a/daggy/include/daggy/DAG.hpp +++ b/daggy/include/daggy/DAG.hpp @@ -1,15 +1,15 @@ #pragma once -#include #include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include #include "Defines.hpp" @@ -20,63 +20,67 @@ namespace daggy { - template - struct Vertex { - RunState state; - uint32_t depCount; - V data; - std::unordered_set children; - }; + template + struct Vertex + { + RunState state; + uint32_t depCount; + V data; + std::unordered_set children; + }; + template + class DAG + { + using Edge = std::pair; - template - class DAG { - using Edge = std::pair; - public: - // Vertices - void addVertex(K id, V data); + public: + // Vertices + void addVertex(K id, V data); - std::unordered_set getVertices() const; + std::unordered_set getVertices() const; - // Edges - void addEdge(const K &src, const K &dst); + // Edges + void addEdge(const K &src, const K &dst); - void addEdgeIf(const K &src, std::function &v)> predicate); + void addEdgeIf(const K &src, + std::function &v)> predicate); - bool isValid() const; + bool isValid() const; - bool hasVertex(const K &from); + bool hasVertex(const K &from); - const std::vector &getEdges(); + const std::vector &getEdges(); - // Attributes - size_t size() const; + // Attributes + size_t size() const; - bool empty() const; + bool empty() const; - // Reset the DAG to completely unvisited - void reset(); + // Reset the DAG to completely unvisited + void reset(); - // Reset any vertex with RUNNING state to QUEUED - void resetRunning(); + // Reset any vertex with RUNNING state to QUEUED + void resetRunning(); - RunState getVertexState(const K &id) const; + RunState getVertexState(const K &id) const; - void setVertexState(const K &id, RunState state); + void setVertexState(const K &id, RunState state); - void forEach(std::function> &)> fun) const; + void forEach( + std::function> &)> fun) const; - bool allVisited() const; + bool allVisited() const; - std::optional> visitNext(); + std::optional> visitNext(); - Vertex &getVertex(const K &id); + Vertex &getVertex(const K &id); - void completeVisit(const K &id); + void completeVisit(const K &id); - private: - std::unordered_map> vertices_; - }; -} + private: + std::unordered_map> vertices_; + }; +} // namespace daggy #include "DAG.impl.hxx" diff --git a/daggy/include/daggy/DAG.impl.hxx b/daggy/include/daggy/DAG.impl.hxx index 527ac4b..79da631 100644 --- a/daggy/include/daggy/DAG.impl.hxx +++ b/daggy/include/daggy/DAG.impl.hxx @@ -1,148 +1,183 @@ namespace daggy { - template - size_t DAG::size() const { return vertices_.size(); } + template + size_t DAG::size() const + { + return vertices_.size(); + } - template - bool DAG::empty() const { return vertices_.empty(); } + template + bool DAG::empty() const + { + return vertices_.empty(); + } - template - bool DAG::hasVertex(const K &id) { return vertices_.count(id) != 0; } + template + bool DAG::hasVertex(const K &id) + { + return vertices_.count(id) != 0; + } - template - Vertex &DAG::getVertex(const K &id) { return vertices_.at(id); } + template + Vertex &DAG::getVertex(const K &id) + { + return vertices_.at(id); + } - template - std::unordered_set DAG::getVertices() const { - std::unordered_set keys; - for (const auto it : vertices_) { - keys.insert(it.first); - } - return keys; + template + std::unordered_set DAG::getVertices() const + { + std::unordered_set keys; + for (const auto it : vertices_) { + keys.insert(it.first); + } + return keys; + } + + template + void DAG::addVertex(K id, V data) + { + if (vertices_.count(id) != 0) { + std::stringstream ss; + ss << "A vertex with ID " << id << " already exists in the DAG"; + throw std::runtime_error(ss.str()); + } + vertices_.emplace( + id, + Vertex{.state = RunState::QUEUED, .depCount = 0, .data = data}); + } + + template + void DAG::addEdge(const K &from, const K &to) + { + if (vertices_.find(from) == vertices_.end()) + throw std::runtime_error("No such vertex"); + if (vertices_.find(to) == vertices_.end()) + throw std::runtime_error("No such vertex"); + vertices_.at(from).children.insert(to); + vertices_.at(to).depCount++; + } + + template + void DAG::addEdgeIf( + const K &src, std::function &v)> predicate) + { + auto &parent = vertices_.at(src); + for (auto &[name, vertex] : vertices_) { + if (!predicate(vertex)) + continue; + if (name == src) + continue; + parent.children.insert(name); + vertex.depCount++; + } + } + + template + bool DAG::isValid() const + { + std::unordered_map depCounts; + std::queue ready; + size_t processed = 0; + + for (const auto &[k, v] : vertices_) { + depCounts[k] = v.depCount; + if (v.depCount == 0) + ready.push(k); } - template - void DAG::addVertex(K id, V data) { - if (vertices_.count(id) != 0) { - std::stringstream ss; - ss << "A vertex with ID " << id << " already exists in the DAG"; - throw std::runtime_error(ss.str()); - } - vertices_.emplace(id, Vertex{.state = RunState::QUEUED, .depCount = 0, .data = data}); + while (!ready.empty()) { + const auto &k = ready.front(); + for (const auto &child : vertices_.at(k).children) { + auto dc = --depCounts[child]; + if (dc == 0) + ready.push(child); + } + processed++; + ready.pop(); } - template - void DAG::addEdge(const K &from, const K &to) { - if (vertices_.find(from) == vertices_.end()) throw std::runtime_error("No such vertex"); - if (vertices_.find(to) == vertices_.end()) throw std::runtime_error("No such vertex"); - vertices_.at(from).children.insert(to); - vertices_.at(to).depCount++; + return processed == vertices_.size(); + } + + template + void DAG::reset() + { + // Reset the state of all vertices + for (auto &[_, v] : vertices_) { + v.state = RunState::QUEUED; + v.depCount = 0; } - template - void DAG::addEdgeIf(const K &src, std::function &v)> predicate) { - auto & parent = vertices_.at(src); - for (auto &[name, vertex]: vertices_) { - if (! predicate(vertex)) continue; - if (name == src) continue; - parent.children.insert(name); - vertex.depCount++; - } + // Calculate the upstream count + for (auto &[_, v] : vertices_) { + for (auto c : v.children) { + vertices_.at(c).depCount++; + } } + } - template - bool DAG::isValid() const { - std::unordered_map depCounts; - std::queue ready; - size_t processed = 0; - - for (const auto & [k, v] : vertices_) { - depCounts[k] = v.depCount; - if (v.depCount == 0) ready.push(k); - } - - while (! ready.empty()) { - const auto & k = ready.front(); - for (const auto & child : vertices_.at(k).children) { - auto dc = --depCounts[child]; - if (dc == 0) ready.push(child); - } - processed++; - ready.pop(); - } - - return processed == vertices_.size(); + template + void DAG::resetRunning() + { + for (auto &[k, v] : vertices_) { + if (v.state != +RunState::RUNNING) + continue; + v.state = RunState::QUEUED; } + } - template - void DAG::reset() { - // Reset the state of all vertices - for (auto &[_, v]: vertices_) { - v.state = RunState::QUEUED; - v.depCount = 0; - } + template + void DAG::setVertexState(const K &id, RunState state) + { + vertices_.at(id).state = state; + } - // Calculate the upstream count - for (auto &[_, v]: vertices_) { - for (auto c: v.children) { - vertices_.at(c).depCount++; - } - } + template + bool DAG::allVisited() const + { + for (const auto &[_, v] : vertices_) { + if (v.state != +RunState::COMPLETED) + return false; } + return true; + } - template - void DAG::resetRunning() { - for (auto &[k, v]: vertices_) { - if (v.state != +RunState::RUNNING) continue; - v.state = RunState::QUEUED; - } + template + std::optional> DAG::visitNext() + { + for (auto &[k, v] : vertices_) { + if (v.state != +RunState::QUEUED) + continue; + if (v.depCount != 0) + continue; + v.state = RunState::RUNNING; + return std::make_pair(k, v.data); } + return {}; + } - template - void DAG::setVertexState(const K &id, RunState state) { - vertices_.at(id).state = state; + template + void DAG::completeVisit(const K &id) + { + auto &v = vertices_.at(id); + v.state = RunState::COMPLETED; + for (auto c : v.children) { + --vertices_.at(c).depCount; } + } - template - bool DAG::allVisited() const { - for (const auto &[_, v]: vertices_) { - if (v.state != +RunState::COMPLETED) return false; - } - return true; + template + void DAG::forEach(std::function> &) + + > + fun) const + { + for (auto it = vertices_.begin(); it != vertices_. + + end(); + + ++it) { + fun(*it); } - - template - std::optional> - DAG::visitNext() { - for (auto &[k, v]: vertices_) { - if (v.state != +RunState::QUEUED) continue; - if (v.depCount != 0) continue; - v.state = RunState::RUNNING; - return std::make_pair(k, v.data); - } - return {}; - } - - template - void DAG::completeVisit(const K &id) { - auto &v = vertices_.at(id); - v.state = RunState::COMPLETED; - for (auto c: v.children) { - --vertices_.at(c).depCount; - } - } - - template - void DAG::forEach(std::function> &) - - > fun) const { - for ( - auto it = vertices_.begin(); - it != vertices_. - - end(); - - ++it) { - fun(*it); -} -} -} + } +} // namespace daggy diff --git a/daggy/include/daggy/Defines.hpp b/daggy/include/daggy/Defines.hpp index ba7fb4a..cbf4422 100644 --- a/daggy/include/daggy/Defines.hpp +++ b/daggy/include/daggy/Defines.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -7,60 +9,56 @@ #include #include -#include - namespace daggy { - // Commands and parameters - using ConfigValue = std::variant>; - using ConfigValues = std::unordered_map; - using Command = std::vector; + // Commands and parameters + using ConfigValue = std::variant>; + using ConfigValues = std::unordered_map; + using Command = std::vector; - // Time - using Clock = std::chrono::high_resolution_clock; - using TimePoint = std::chrono::time_point; + // Time + using Clock = std::chrono::high_resolution_clock; + using TimePoint = std::chrono::time_point; - // DAG Runs - using DAGRunID = size_t; + // DAG Runs + using DAGRunID = size_t; - BETTER_ENUM(RunState, uint32_t, - QUEUED = 1 << 0, - RUNNING = 1 << 1, - RETRY = 1 << 2, - ERRORED = 1 << 3, - KILLED = 1 << 4, - COMPLETED = 1 << 5 - ); + BETTER_ENUM(RunState, uint32_t, QUEUED = 1 << 0, RUNNING = 1 << 1, + RETRY = 1 << 2, ERRORED = 1 << 3, KILLED = 1 << 4, + COMPLETED = 1 << 5); - struct Task { - std::string definedName; - bool isGenerator; // True if the output of this task is a JSON set of tasks to complete - uint32_t maxRetries; - uint32_t retryIntervalSeconds; // Time to wait between retries - ConfigValues job; // It's up to the individual inspectors to convert values from strings // array of strings - std::unordered_set children; - std::unordered_set parents; + struct Task + { + std::string definedName; + bool isGenerator; // True if the output of this task is a JSON set of tasks + // to complete + uint32_t maxRetries; + uint32_t retryIntervalSeconds; // Time to wait between retries + ConfigValues job; // It's up to the individual inspectors to convert values + // from strings // array of strings + std::unordered_set children; + std::unordered_set parents; - bool operator==(const Task &other) const { - return (definedName == other.definedName) - and (maxRetries == other.maxRetries) - and (retryIntervalSeconds == other.retryIntervalSeconds) - and (job == other.job) - and (children == other.children) - and (parents == other.parents) - and (isGenerator == other.isGenerator); - } - }; + bool operator==(const Task &other) const + { + return (definedName == other.definedName) and + (maxRetries == other.maxRetries) and + (retryIntervalSeconds == other.retryIntervalSeconds) and + (job == other.job) and (children == other.children) and + (parents == other.parents) and (isGenerator == other.isGenerator); + } + }; - using TaskSet = std::unordered_map; + using TaskSet = std::unordered_map; - struct AttemptRecord { - TimePoint startTime; - TimePoint stopTime; - int rc; // RC from the task - std::string executorLog; // Logs from the dag_executor - std::string outputLog; // stdout from command - std::string errorLog; // stderr from command - }; -} + struct AttemptRecord + { + TimePoint startTime; + TimePoint stopTime; + int rc; // RC from the task + std::string executorLog; // Logs from the dag_executor + std::string outputLog; // stdout from command + std::string errorLog; // stderr from command + }; +} // namespace daggy BETTER_ENUMS_DECLARE_STD_HASH(daggy::RunState) diff --git a/daggy/include/daggy/Serialization.hpp b/daggy/include/daggy/Serialization.hpp index cdea8fe..0d4fb36 100644 --- a/daggy/include/daggy/Serialization.hpp +++ b/daggy/include/daggy/Serialization.hpp @@ -1,45 +1,48 @@ #pragma once -#include -#include -#include -#include - #include +#include +#include +#include +#include + #include "Defines.hpp" namespace rj = rapidjson; namespace daggy { - void checkRJParse(const rj::ParseResult &result, const std::string &prefix = ""); + void checkRJParse(const rj::ParseResult &result, + const std::string &prefix = ""); - // Parameters - ConfigValues configFromJSON(const std::string &jsonSpec); + // Parameters + ConfigValues configFromJSON(const std::string &jsonSpec); - ConfigValues configFromJSON(const rj::Value &spec); + ConfigValues configFromJSON(const rj::Value &spec); - std::string configToJSON(const ConfigValues &config); + std::string configToJSON(const ConfigValues &config); - // Tasks - Task - taskFromJSON(const std::string &name, const rj::Value &spec, const ConfigValues &jobDefaults = {}); + // Tasks + Task taskFromJSON(const std::string &name, const rj::Value &spec, + const ConfigValues &jobDefaults = {}); - TaskSet tasksFromJSON(const std::string &jsonSpec, const ConfigValues &jobDefaults = {}); + TaskSet tasksFromJSON(const std::string &jsonSpec, + const ConfigValues &jobDefaults = {}); - TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults = {}); + TaskSet tasksFromJSON(const rj::Value &spec, + const ConfigValues &jobDefaults = {}); - std::string taskToJSON(const Task &task); + std::string taskToJSON(const Task &task); - std::string tasksToJSON(const TaskSet &tasks); + std::string tasksToJSON(const TaskSet &tasks); - // Attempt Records - std::string attemptRecordToJSON(const AttemptRecord &attemptRecord); + // Attempt Records + std::string attemptRecordToJSON(const AttemptRecord &attemptRecord); - // default serialization - std::ostream &operator<<(std::ostream &os, const Task &task); + // default serialization + std::ostream &operator<<(std::ostream &os, const Task &task); - std::string timePointToString(const TimePoint &tp); + std::string timePointToString(const TimePoint &tp); - TimePoint stringToTimePoint(const std::string &timeStr); -} + TimePoint stringToTimePoint(const std::string &timeStr); +} // namespace daggy diff --git a/daggy/include/daggy/Server.hpp b/daggy/include/daggy/Server.hpp index 4d044a0..9e36503 100644 --- a/daggy/include/daggy/Server.hpp +++ b/daggy/include/daggy/Server.hpp @@ -1,58 +1,68 @@ #pragma once -#include - #include #include #include +#include + #include "ThreadPool.hpp" -#include "loggers/dag_run/DAGRunLogger.hpp" #include "executors/task/TaskExecutor.hpp" +#include "loggers/dag_run/DAGRunLogger.hpp" namespace fs = std::filesystem; namespace daggy { - class Server { - public: - Server(const Pistache::Address &listenSpec, loggers::dag_run::DAGRunLogger &logger, - executors::task::TaskExecutor &executor, - size_t nDAGRunners - ) - : endpoint_(listenSpec), desc_("Daggy API", "0.1"), logger_(logger), executor_(executor), - runnerPool_(nDAGRunners) {} + class Server + { + public: + Server(const Pistache::Address &listenSpec, + loggers::dag_run::DAGRunLogger &logger, + executors::task::TaskExecutor &executor, size_t nDAGRunners) + : endpoint_(listenSpec) + , desc_("Daggy API", "0.1") + , logger_(logger) + , executor_(executor) + , runnerPool_(nDAGRunners) + { + } - Server &setWebHandlerThreads(size_t nThreads); + Server &setWebHandlerThreads(size_t nThreads); - Server &setSSLCertificates(const fs::path &cert, const fs::path &key); + Server &setSSLCertificates(const fs::path &cert, const fs::path &key); - void init(int threads = 1); + void init(int threads = 1); - void start(); + void start(); - uint16_t getPort() const; + uint16_t getPort() const; - void shutdown(); + void shutdown(); - private: - void createDescription(); + private: + void createDescription(); - void handleRunDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void handleRunDAG(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response); - void handleGetDAGRuns(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void handleGetDAGRuns(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response); - void handleGetDAGRun(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void handleGetDAGRun(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response); - void handleReady(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response); + void handleReady(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response); - bool handleAuth(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response); + bool handleAuth(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter &response); - Pistache::Http::Endpoint endpoint_; - Pistache::Rest::Description desc_; - Pistache::Rest::Router router_; + Pistache::Http::Endpoint endpoint_; + Pistache::Rest::Description desc_; + Pistache::Rest::Router router_; - loggers::dag_run::DAGRunLogger &logger_; - executors::task::TaskExecutor &executor_; - ThreadPool runnerPool_; - }; -} + loggers::dag_run::DAGRunLogger &logger_; + executors::task::TaskExecutor &executor_; + ThreadPool runnerPool_; + }; +} // namespace daggy diff --git a/daggy/include/daggy/ThreadPool.hpp b/daggy/include/daggy/ThreadPool.hpp index 887441b..b6ade2a 100644 --- a/daggy/include/daggy/ThreadPool.hpp +++ b/daggy/include/daggy/ThreadPool.hpp @@ -1,165 +1,186 @@ #pragma once #include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include using namespace std::chrono_literals; namespace daggy { - /* - A Task Queue is a collection of async tasks to be executed by the - thread pool. Using individual task queues allows for a rough QoS - when a single thread may be submitting batches of requests -- - one producer won't starve out another, but all tasks will be run - as quickly as possible. - */ - class TaskQueue { - public: - template - decltype(auto) addTask(F &&f, Args &&... args) { - // using return_type = std::invoke_result::type; - using return_type = std::invoke_result_t; + /* + A Task Queue is a collection of async tasks to be executed by the + thread pool. Using individual task queues allows for a rough QoS + when a single thread may be submitting batches of requests -- + one producer won't starve out another, but all tasks will be run + as quickly as possible. + */ + class TaskQueue + { + public: + template + decltype(auto) addTask(F &&f, Args &&...args) + { + // using return_type = std::invoke_result::type; + using return_type = std::invoke_result_t; - std::packaged_task task( - std::bind(std::forward(f), std::forward(args)...) - ); + std::packaged_task task( + std::bind(std::forward(f), std::forward(args)...)); - std::future res = task.get_future(); + std::future res = task.get_future(); + { + std::lock_guard guard(mtx_); + tasks_.emplace(std::move(task)); + } + return res; + } + + std::packaged_task pop() + { + std::lock_guard guard(mtx_); + auto task = std::move(tasks_.front()); + tasks_.pop(); + return task; + } + + size_t size() + { + std::lock_guard guard(mtx_); + return tasks_.size(); + } + + bool empty() + { + std::lock_guard guard(mtx_); + return tasks_.empty(); + } + + private: + std::queue> tasks_; + std::mutex mtx_; + }; + + class ThreadPool + { + public: + explicit ThreadPool(size_t nWorkers) + : tqit_(taskQueues_.begin()) + , stop_(false) + , drain_(false) + { + resize(nWorkers); + } + + ~ThreadPool() + { + shutdown(); + } + + void shutdown() + { + stop_ = true; + cv_.notify_all(); + for (std::thread &worker : workers_) { + if (worker.joinable()) + worker.join(); + } + } + + void drain() + { + drain_ = true; + while (true) { + { + std::lock_guard guard(mtx_); + if (taskQueues_.empty()) + break; + } + std::this_thread::sleep_for(250ms); + } + } + + void restart() + { + drain_ = false; + } + + void resize(size_t nWorkers) + { + shutdown(); + workers_.clear(); + stop_ = false; + + for (size_t i = 0; i < nWorkers; ++i) + workers_.emplace_back([&] { + while (true) { + std::packaged_task task; { - std::lock_guard guard(mtx_); - tasks_.emplace(std::move(task)); + std::unique_lock lock(mtx_); + cv_.wait(lock, [&] { return stop_ || !taskQueues_.empty(); }); + if (taskQueues_.empty()) { + if (stop_) + return; + continue; + } + if (tqit_ == taskQueues_.end()) + tqit_ = taskQueues_.begin(); + task = std::move((*tqit_)->pop()); + if ((*tqit_)->empty()) { + tqit_ = taskQueues_.erase(tqit_); + } + else { + tqit_++; + } } - return res; - } - - std::packaged_task pop() { - std::lock_guard guard(mtx_); - auto task = std::move(tasks_.front()); - tasks_.pop(); - return task; - } - - size_t size() { - std::lock_guard guard(mtx_); - return tasks_.size(); - } - - bool empty() { - std::lock_guard guard(mtx_); - return tasks_.empty(); - } - - private: - std::queue > tasks_; - std::mutex mtx_; + task(); + } + }); }; - class ThreadPool { - public: - explicit ThreadPool(size_t nWorkers) - : - tqit_(taskQueues_.begin()), stop_(false), drain_(false) { - resize(nWorkers); - } + template + decltype(auto) addTask(F &&f, Args &&...args) + { + if (drain_) + throw std::runtime_error("Unable to add task to draining pool"); + auto tq = std::make_shared(); - ~ThreadPool() { shutdown(); } + auto fut = tq->addTask(f, args...); - void shutdown() { - stop_ = true; - cv_.notify_all(); - for (std::thread &worker : workers_) { - if (worker.joinable()) - worker.join(); - } - } + { + std::lock_guard guard(mtx_); + taskQueues_.push_back(tq); + } + cv_.notify_one(); + return fut; + } - void drain() { - drain_ = true; - while (true) { - { - std::lock_guard guard(mtx_); - if (taskQueues_.empty()) break; - } - std::this_thread::sleep_for(250ms); - } - } + void addTasks(std::shared_ptr tq) + { + if (drain_) + throw std::runtime_error("Unable to add task to draining pool"); + std::lock_guard guard(mtx_); + taskQueues_.push_back(tq); + cv_.notify_one(); + } - void restart() { - drain_ = false; - } + private: + // need to keep track of threads so we can join them + std::vector workers_; + // the task queue + std::list> taskQueues_; + std::list>::iterator tqit_; - void resize(size_t nWorkers) { - shutdown(); - workers_.clear(); - stop_ = false; + // synchronization + std::mutex mtx_; + std::condition_variable cv_; + std::atomic stop_; + std::atomic drain_; + }; - for (size_t i = 0; i < nWorkers; ++i) - workers_.emplace_back([&] { - while (true) { - std::packaged_task task; - { - std::unique_lock lock(mtx_); - cv_.wait(lock, [&] { return stop_ || !taskQueues_.empty(); }); - if (taskQueues_.empty()) { - if (stop_) return; - continue; - } - if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin(); - task = std::move((*tqit_)->pop()); - if ((*tqit_)->empty()) { - tqit_ = taskQueues_.erase(tqit_); - } else { - tqit_++; - } - } - task(); - } - } - ); - }; - - template - decltype(auto) addTask(F &&f, Args &&... args) { - if (drain_) throw std::runtime_error("Unable to add task to draining pool"); - auto tq = std::make_shared(); - - auto fut = tq->addTask(f, args...); - - { - std::lock_guard guard(mtx_); - taskQueues_.push_back(tq); - } - cv_.notify_one(); - return fut; - } - - void addTasks(std::shared_ptr tq) { - if (drain_) throw std::runtime_error("Unable to add task to draining pool"); - std::lock_guard guard(mtx_); - taskQueues_.push_back(tq); - cv_.notify_one(); - } - - private: - // need to keep track of threads so we can join them - std::vector workers_; - // the task queue - std::list> taskQueues_; - std::list>::iterator tqit_; - - // synchronization - std::mutex mtx_; - std::condition_variable cv_; - std::atomic stop_; - std::atomic drain_; - }; - -} +} // namespace daggy diff --git a/daggy/include/daggy/Utilities.hpp b/daggy/include/daggy/Utilities.hpp index 7e70626..f5e9602 100644 --- a/daggy/include/daggy/Utilities.hpp +++ b/daggy/include/daggy/Utilities.hpp @@ -1,41 +1,39 @@ #pragma once -#include -#include -#include -#include - #include -#include "daggy/loggers/dag_run/DAGRunLogger.hpp" -#include "daggy/executors/task/TaskExecutor.hpp" -#include "Defines.hpp" +#include +#include +#include +#include + #include "DAG.hpp" +#include "Defines.hpp" +#include "daggy/executors/task/TaskExecutor.hpp" +#include "daggy/loggers/dag_run/DAGRunLogger.hpp" namespace daggy { - using TaskDAG = DAG; + using TaskDAG = DAG; - std::string globalSub(std::string string, const std::string &pattern, const std::string &replacement); + std::string globalSub(std::string string, const std::string &pattern, + const std::string &replacement); - std::vector interpolateValues(const std::vector &raw, const ConfigValues &values); + std::vector interpolateValues(const std::vector &raw, + const ConfigValues &values); - TaskSet - expandTaskSet(const TaskSet &tasks, - executors::task::TaskExecutor &executor, - const ConfigValues &interpolatedValues = {}); + TaskSet expandTaskSet(const TaskSet &tasks, + executors::task::TaskExecutor &executor, + const ConfigValues &interpolatedValues = {}); + TaskDAG buildDAGFromTasks( + TaskSet &tasks, + const std::vector &updates = {}); - TaskDAG - buildDAGFromTasks(TaskSet &tasks, - const std::vector &updates = {}); + void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks); - void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks); + TaskDAG runDAG(DAGRunID runID, executors::task::TaskExecutor &executor, + loggers::dag_run::DAGRunLogger &logger, TaskDAG dag, + const ConfigValues job = {}); - TaskDAG runDAG(DAGRunID runID, - executors::task::TaskExecutor &executor, - loggers::dag_run::DAGRunLogger &logger, - TaskDAG dag, - const ConfigValues job = {}); - - std::ostream &operator<<(std::ostream &os, const TimePoint &tp); -} + std::ostream &operator<<(std::ostream &os, const TimePoint &tp); +} // namespace daggy diff --git a/daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp b/daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp index 897dc94..2272f98 100644 --- a/daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp +++ b/daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp @@ -1,27 +1,33 @@ #pragma once -#include "TaskExecutor.hpp" #include +#include "TaskExecutor.hpp" + namespace daggy::executors::task { - class ForkingTaskExecutor : public TaskExecutor { - public: - using Command = std::vector; + class ForkingTaskExecutor : public TaskExecutor + { + public: + using Command = std::vector; - ForkingTaskExecutor(size_t nThreads) - : tp_(nThreads) {} + ForkingTaskExecutor(size_t nThreads) + : tp_(nThreads) + { + } - // Validates the job to ensure that all required values are set and are of the right type, - bool validateTaskParameters(const ConfigValues &job) override; + // Validates the job to ensure that all required values are set and are of + // the right type, + bool validateTaskParameters(const ConfigValues &job) override; - std::vector - expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; + std::vector expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) override; - // Runs the task - std::future execute(const std::string &taskName, const Task &task) override; + // Runs the task + std::future execute(const std::string &taskName, + const Task &task) override; - private: - ThreadPool tp_; - AttemptRecord runTask(const Task & task); - }; -} + private: + ThreadPool tp_; + AttemptRecord runTask(const Task &task); + }; +} // namespace daggy::executors::task diff --git a/daggy/include/daggy/executors/task/NoopTaskExecutor.hpp b/daggy/include/daggy/executors/task/NoopTaskExecutor.hpp index 4455dfd..751b255 100644 --- a/daggy/include/daggy/executors/task/NoopTaskExecutor.hpp +++ b/daggy/include/daggy/executors/task/NoopTaskExecutor.hpp @@ -3,18 +3,20 @@ #include "TaskExecutor.hpp" namespace daggy::executors::task { - class NoopTaskExecutor : public TaskExecutor { - public: - using Command = std::vector; + class NoopTaskExecutor : public TaskExecutor + { + public: + using Command = std::vector; - // Validates the job to ensure that all required values are set and are of the right type, - bool validateTaskParameters(const ConfigValues &job) override; + // Validates the job to ensure that all required values are set and are of + // the right type, + bool validateTaskParameters(const ConfigValues &job) override; - std::vector - expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; - - // Runs the task - std::future execute(const std::string &taskName, const Task &task) override; - }; -} + std::vector expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) override; + // Runs the task + std::future execute(const std::string &taskName, + const Task &task) override; + }; +} // namespace daggy::executors::task diff --git a/daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp b/daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp index 09145e1..db3cbdf 100644 --- a/daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp +++ b/daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp @@ -3,35 +3,39 @@ #include "TaskExecutor.hpp" namespace daggy::executors::task { - class SlurmTaskExecutor : public TaskExecutor { - public: - using Command = std::vector; + class SlurmTaskExecutor : public TaskExecutor + { + public: + using Command = std::vector; - SlurmTaskExecutor(); - ~SlurmTaskExecutor(); + SlurmTaskExecutor(); + ~SlurmTaskExecutor(); - // Validates the job to ensure that all required values are set and are of the right type, - bool validateTaskParameters(const ConfigValues &job) override; + // Validates the job to ensure that all required values are set and are of + // the right type, + bool validateTaskParameters(const ConfigValues &job) override; - std::vector - expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) override; + std::vector expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) override; - // Runs the task - std::future execute(const std::string &taskName, const Task &task) override; + // Runs the task + std::future execute(const std::string &taskName, + const Task &task) override; - private: - struct Job { - std::promise prom; - std::string stdoutFile; - std::string stderrFile; - }; - - std::mutex promiseGuard_; - std::unordered_map runningJobs_; - std::atomic running_; - - // Monitors jobs and resolves promises - std::thread monitorWorker_; - void monitor(); + private: + struct Job + { + std::promise prom; + std::string stdoutFile; + std::string stderrFile; }; -} + + std::mutex promiseGuard_; + std::unordered_map runningJobs_; + std::atomic running_; + + // Monitors jobs and resolves promises + std::thread monitorWorker_; + void monitor(); + }; +} // namespace daggy::executors::task diff --git a/daggy/include/daggy/executors/task/TaskExecutor.hpp b/daggy/include/daggy/executors/task/TaskExecutor.hpp index afed3d9..682bcea 100644 --- a/daggy/include/daggy/executors/task/TaskExecutor.hpp +++ b/daggy/include/daggy/executors/task/TaskExecutor.hpp @@ -1,31 +1,33 @@ #pragma once #include +#include #include #include #include #include -#include - /* Executors run Tasks, returning a future with the results. If there are many retries, logs are returned for each attempt. */ namespace daggy::executors::task { - class TaskExecutor { - public: - virtual ~TaskExecutor() = default; + class TaskExecutor + { + public: + virtual ~TaskExecutor() = default; - // Validates the job to ensure that all required values are set and are of the right type, - virtual bool validateTaskParameters(const ConfigValues &job) = 0; + // Validates the job to ensure that all required values are set and are of + // the right type, + virtual bool validateTaskParameters(const ConfigValues &job) = 0; - // Will use the expansion values to return the fully expanded tasks. - virtual std::vector - expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) = 0; + // Will use the expansion values to return the fully expanded tasks. + virtual std::vector expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) = 0; - // Blocking execution of a task - virtual std::future execute(const std::string &taskName, const Task &task) = 0; - }; -} + // Blocking execution of a task + virtual std::future execute(const std::string &taskName, + const Task &task) = 0; + }; +} // namespace daggy::executors::task diff --git a/daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp b/daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp index d8ed8c7..a1b211e 100644 --- a/daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp +++ b/daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp @@ -11,32 +11,32 @@ be supported. */ -namespace daggy { - namespace loggers { - namespace dag_run { - class DAGRunLogger { - public: - virtual ~DAGRunLogger() = default; +namespace daggy { namespace loggers { namespace dag_run { + class DAGRunLogger + { + public: + virtual ~DAGRunLogger() = default; - // Execution - virtual DAGRunID startDAGRun(std::string name, const TaskSet &tasks) = 0; + // Execution + virtual DAGRunID startDAGRun(std::string name, const TaskSet &tasks) = 0; - virtual void addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) = 0; + virtual void addTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) = 0; - virtual void updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) = 0; + virtual void updateTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) = 0; - virtual void updateDAGRunState(DAGRunID dagRunID, RunState state) = 0; + virtual void updateDAGRunState(DAGRunID dagRunID, RunState state) = 0; - virtual void - logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, const AttemptRecord &attempt) = 0; + virtual void logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, + const AttemptRecord &attempt) = 0; - virtual void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) = 0; + virtual void updateTaskState(DAGRunID dagRunID, const std::string &taskName, + RunState state) = 0; - // Querying - virtual std::vector getDAGs(uint32_t stateMask) = 0; + // Querying + virtual std::vector getDAGs(uint32_t stateMask) = 0; - virtual DAGRunRecord getDAGRun(DAGRunID dagRunID) = 0; - }; - } - } -} + virtual DAGRunRecord getDAGRun(DAGRunID dagRunID) = 0; + }; +}}} // namespace daggy::loggers::dag_run diff --git a/daggy/include/daggy/loggers/dag_run/Defines.hpp b/daggy/include/daggy/loggers/dag_run/Defines.hpp index a18fc94..eff8010 100644 --- a/daggy/include/daggy/loggers/dag_run/Defines.hpp +++ b/daggy/include/daggy/loggers/dag_run/Defines.hpp @@ -2,40 +2,44 @@ #include #include -#include -#include #include +#include +#include #include "../../Defines.hpp" namespace daggy::loggers::dag_run { - struct TaskUpdateRecord { - TimePoint time; - std::string taskName; - RunState newState; - }; + struct TaskUpdateRecord + { + TimePoint time; + std::string taskName; + RunState newState; + }; - struct DAGUpdateRecord { - TimePoint time; - RunState newState; - }; + struct DAGUpdateRecord + { + TimePoint time; + RunState newState; + }; - // Pretty heavy weight, but - struct DAGRunRecord { - std::string name; - TaskSet tasks; - std::unordered_map taskRunStates; - std::unordered_map> taskAttempts; - std::vector taskStateChanges; - std::vector dagStateChanges; - }; + // Pretty heavy weight, but + struct DAGRunRecord + { + std::string name; + TaskSet tasks; + std::unordered_map taskRunStates; + std::unordered_map> taskAttempts; + std::vector taskStateChanges; + std::vector dagStateChanges; + }; - struct DAGRunSummary { - DAGRunID runID; - std::string name; - RunState runState; - TimePoint startTime; - TimePoint lastUpdate; - std::unordered_map taskStateCounts; - }; -} + struct DAGRunSummary + { + DAGRunID runID; + std::string name; + RunState runState; + TimePoint startTime; + TimePoint lastUpdate; + std::unordered_map taskStateCounts; + }; +} // namespace daggy::loggers::dag_run diff --git a/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp b/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp index a34c5a7..66e7795 100644 --- a/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp +++ b/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp @@ -1,10 +1,11 @@ #pragma once -#include +#include + #include +#include #include -#include #include "DAGRunLogger.hpp" #include "Defines.hpp" @@ -12,56 +13,58 @@ namespace fs = std::filesystem; namespace rj = rapidjson; namespace daggy::loggers::dag_run { - /* - * This logger should only be used for debug purposes. It's not really optimized for querying, and will - * use a ton of inodes to track state. - * - * On the plus side, it's trivial to look at without using the API. - * - * Filesystem logger creates the following structure: - * {root}/ - * runs/ - * {runID}/ - * meta.json --- Contains the DAG name, task definitions - * states.csv --- DAG state changes - * {taskName}/ - * states.csv --- TASK state changes - * {attempt}/ - * metadata.json --- timestamps and rc - * output.log - * error.log - * executor.log - */ - class FileSystemLogger : public DAGRunLogger { - public: - FileSystemLogger(fs::path root); + /* + * This logger should only be used for debug purposes. It's not really + * optimized for querying, and will use a ton of inodes to track state. + * + * On the plus side, it's trivial to look at without using the API. + * + * Filesystem logger creates the following structure: + * {root}/ + * runs/ + * {runID}/ + * meta.json --- Contains the DAG name, task definitions + * states.csv --- DAG state changes + * {taskName}/ + * states.csv --- TASK state changes + * {attempt}/ + * metadata.json --- timestamps and rc + * output.log + * error.log + * executor.log + */ + class FileSystemLogger : public DAGRunLogger + { + public: + FileSystemLogger(fs::path root); - // Execution - DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; + // Execution + DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; - void updateDAGRunState(DAGRunID dagRunID, RunState state) override; + void updateDAGRunState(DAGRunID dagRunID, RunState state) override; - void - logTaskAttempt(DAGRunID, const std::string &taskName, const AttemptRecord &attempt) override; + void logTaskAttempt(DAGRunID, const std::string &taskName, + const AttemptRecord &attempt) override; - void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) override; + void updateTaskState(DAGRunID dagRunID, const std::string &taskName, + RunState state) override; - // Querying - std::vector getDAGs(uint32_t stateMask) override; + // Querying + std::vector getDAGs(uint32_t stateMask) override; - DAGRunRecord getDAGRun(DAGRunID dagRunID) override; + DAGRunRecord getDAGRun(DAGRunID dagRunID) override; - private: - fs::path root_; - std::atomic nextRunID_; - std::mutex lock_; + private: + fs::path root_; + std::atomic nextRunID_; + std::mutex lock_; - // std::unordered_map runLocks; + // std::unordered_map runLocks; - inline const fs::path getCurrentPath() const; + inline const fs::path getCurrentPath() const; - inline const fs::path getRunsRoot() const; + inline const fs::path getRunsRoot() const; - inline const fs::path getRunRoot(DAGRunID runID) const; - }; -} + inline const fs::path getRunRoot(DAGRunID runID) const; + }; +} // namespace daggy::loggers::dag_run diff --git a/daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp b/daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp index 13c06c0..0948b71 100644 --- a/daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp +++ b/daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp @@ -6,45 +6,46 @@ #include "DAGRunLogger.hpp" #include "Defines.hpp" -namespace daggy { - namespace loggers { - namespace dag_run { - /* - * This logger should only be used for debug purposes. It doesn't actually log anything, just prints stuff - * to stdout. - */ - class OStreamLogger : public DAGRunLogger { - public: - OStreamLogger(std::ostream &os); +namespace daggy { namespace loggers { namespace dag_run { + /* + * This logger should only be used for debug purposes. It doesn't actually log + * anything, just prints stuff to stdout. + */ + class OStreamLogger : public DAGRunLogger + { + public: + OStreamLogger(std::ostream &os); - // Execution - DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; + // Execution + DAGRunID startDAGRun(std::string name, const TaskSet &tasks) override; - void addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) override; + void addTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) override; - void updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) override; + void updateTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) override; - void updateDAGRunState(DAGRunID dagRunID, RunState state) override; + void updateDAGRunState(DAGRunID dagRunID, RunState state) override; - void - logTaskAttempt(DAGRunID, const std::string &taskName, const AttemptRecord &attempt) override; + void logTaskAttempt(DAGRunID, const std::string &taskName, + const AttemptRecord &attempt) override; - void updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) override; + void updateTaskState(DAGRunID dagRunID, const std::string &taskName, + RunState state) override; - // Querying - std::vector getDAGs(uint32_t stateMask) override; + // Querying + std::vector getDAGs(uint32_t stateMask) override; - DAGRunRecord getDAGRun(DAGRunID dagRunID) override; + DAGRunRecord getDAGRun(DAGRunID dagRunID) override; - private: - std::mutex guard_; - std::ostream &os_; - std::vector dagRuns_; + private: + std::mutex guard_; + std::ostream &os_; + std::vector dagRuns_; - void _updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state); + void _updateTaskState(DAGRunID dagRunID, const std::string &taskName, + RunState state); - void _updateDAGRunState(DAGRunID dagRunID, RunState state); - }; - } - } -} + void _updateDAGRunState(DAGRunID dagRunID, RunState state); + }; +}}} // namespace daggy::loggers::dag_run diff --git a/daggy/src/Serialization.cpp b/daggy/src/Serialization.cpp index 5078bba..8f5eac3 100644 --- a/daggy/src/Serialization.cpp +++ b/daggy/src/Serialization.cpp @@ -1,252 +1,292 @@ -#include -#include - #include #include #include +#include +#include namespace daggy { - void checkRJParse(const rj::ParseResult &result, const std::string &prefix) { - if (!result) { - std::stringstream ss; - ss << (prefix.empty() ? "" : prefix + ':') - << "Error parsing JSON: " << rj::GetParseError_En(result.Code()) - << " at byte offset " << result.Offset(); - throw std::runtime_error(ss.str()); + void checkRJParse(const rj::ParseResult &result, const std::string &prefix) + { + if (!result) { + std::stringstream ss; + ss << (prefix.empty() ? "" : prefix + ':') + << "Error parsing JSON: " << rj::GetParseError_En(result.Code()) + << " at byte offset " << result.Offset(); + throw std::runtime_error(ss.str()); + } + } + + ConfigValues configFromJSON(const std::string &jsonSpec) + { + rj::Document doc; + checkRJParse(doc.Parse(jsonSpec.c_str()), "Parsing config"); + return configFromJSON(doc); + } + + ConfigValues configFromJSON(const rj::Value &spec) + { + std::unordered_map parameters; + if (!spec.IsObject()) { + throw std::runtime_error("Parameters in spec is not a JSON dictionary"); + } + for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) { + if (!it->name.IsString()) { + throw std::runtime_error("All keys must be strings."); + } + std::string name = it->name.GetString(); + if (it->value.IsArray()) { + std::vector values; + for (size_t i = 0; i < it->value.Size(); ++i) { + if (!it->value[i].IsString()) { + throw std::runtime_error( + "Attribute for " + std::string{it->name.GetString()} + + " item " + std::to_string(i) + " is not a string."); + } + values.emplace_back(it->value[i].GetString()); } + parameters[name] = values; + } + else if (it->value.IsString()) { + parameters[name] = it->value.GetString(); + } + else { + throw std::runtime_error("Attribute for " + + std::string{it->name.GetString()} + + " is not a string or an array."); + } + } + return parameters; + } + + std::string configToJSON(const ConfigValues &config) + { + std::stringstream ss; + ss << '{'; + bool first = true; + for (const auto &[k, v] : config) { + if (first) { + first = false; + } + else { + ss << ", "; + } + ss << std::quoted(k) << ": "; + if (std::holds_alternative(v)) { + ss << std::quoted(std::get(v)); + } + else { + ss << '['; + const auto &vals = std::get>(v); + bool firstVal = true; + for (const auto &val : vals) { + if (firstVal) { + firstVal = false; + } + else { + ss << ", "; + } + ss << std::quoted(val); + } + ss << ']'; + } + } + ss << '}'; + return ss.str(); + } + + Task taskFromJSON(const std::string &name, const rj::Value &spec, + const ConfigValues &jobDefaults) + { + Task task{.definedName = name, + .isGenerator = false, + .maxRetries = 0, + .retryIntervalSeconds = 0, + .job = jobDefaults}; + if (!spec.IsObject()) { + throw std::runtime_error("Tasks is not an object"); } - ConfigValues configFromJSON(const std::string &jsonSpec) { - rj::Document doc; - checkRJParse(doc.Parse(jsonSpec.c_str()), "Parsing config"); - return configFromJSON(doc); + // Grab the standard fields with defaults; + if (spec.HasMember("isGenerator")) { + task.isGenerator = spec["isGenerator"].GetBool(); } - ConfigValues configFromJSON(const rj::Value &spec) { - std::unordered_map parameters; - if (!spec.IsObject()) { throw std::runtime_error("Parameters in spec is not a JSON dictionary"); } - for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) { - if (!it->name.IsString()) { - throw std::runtime_error("All keys must be strings."); - } - std::string name = it->name.GetString(); - if (it->value.IsArray()) { - std::vector values; - for (size_t i = 0; i < it->value.Size(); ++i) { - if (!it->value[i].IsString()) { - throw std::runtime_error( - "Attribute for " + std::string{it->name.GetString()} + " item " + std::to_string(i) + - " is not a string."); - } - values.emplace_back(it->value[i].GetString()); - } - parameters[name] = values; - } else if (it->value.IsString()) { - parameters[name] = it->value.GetString(); - } else { - throw std::runtime_error( - "Attribute for " + std::string{it->name.GetString()} + " is not a string or an array."); - } - } - return parameters; + if (spec.HasMember("maxRetries")) { + task.maxRetries = spec["maxRetries"].GetInt(); } - std::string configToJSON(const ConfigValues &config) { - std::stringstream ss; - ss << '{'; - bool first = true; - for (const auto &[k, v]: config) { - if (first) { first = false; } else { ss << ", "; } - ss << std::quoted(k) << ": "; - if (std::holds_alternative(v)) { - ss << std::quoted(std::get(v)); - } else { - ss << '['; - const auto &vals = std::get>(v); - bool firstVal = true; - for (const auto &val: vals) { - if (firstVal) { firstVal = false; } else { ss << ", "; } - ss << std::quoted(val); - } - ss << ']'; - } - } - ss << '}'; - return ss.str(); + if (spec.HasMember("retryIntervalSeconds")) { + task.retryIntervalSeconds = spec["retryIntervalSeconds"].GetInt(); } - Task - taskFromJSON(const std::string &name, const rj::Value &spec, const ConfigValues &jobDefaults) { - Task task{ - .definedName = name, - .isGenerator = false, - .maxRetries = 0, - .retryIntervalSeconds = 0, - .job = jobDefaults - }; - if (!spec.IsObject()) { throw std::runtime_error("Tasks is not an object"); } - - // Grab the standard fields with defaults; - if (spec.HasMember("isGenerator")) { - task.isGenerator = spec["isGenerator"].GetBool(); - } - - if (spec.HasMember("maxRetries")) { - task.maxRetries = spec["maxRetries"].GetInt(); - } - - if (spec.HasMember("retryIntervalSeconds")) { - task.retryIntervalSeconds = spec["retryIntervalSeconds"].GetInt(); - } - - // Children / parents - if (spec.HasMember("children")) { - const auto &specChildren = spec["children"].GetArray(); - for (size_t c = 0; c < specChildren.Size(); ++c) { - task.children.insert(specChildren[c].GetString()); - } - } - - if (spec.HasMember("parents")) { - const auto &specParents = spec["parents"].GetArray(); - for (size_t c = 0; c < specParents.Size(); ++c) { - task.parents.insert(specParents[c].GetString()); - } - } - - if (spec.HasMember("job")) { - const auto ¶ms = spec["job"]; - if (!params.IsObject()) throw std::runtime_error("job is not a dictionary."); - for (auto it = params.MemberBegin(); it != params.MemberEnd(); ++it) { - if (!it->name.IsString()) throw std::runtime_error("job key must be a string."); - if (it->value.IsArray()) { - std::vector values; - for (size_t i = 0; i < it->value.Size(); ++i) { - values.emplace_back(it->value[i].GetString()); - } - task.job.insert_or_assign(it->name.GetString(), values); - } else { - task.job.insert_or_assign(it->name.GetString(), it->value.GetString()); - } - } - } - - return task; + // Children / parents + if (spec.HasMember("children")) { + const auto &specChildren = spec["children"].GetArray(); + for (size_t c = 0; c < specChildren.Size(); ++c) { + task.children.insert(specChildren[c].GetString()); + } } - TaskSet tasksFromJSON(const std::string &jsonSpec, const ConfigValues &jobDefaults) { - rj::Document doc; - checkRJParse(doc.Parse(jsonSpec.c_str())); - return tasksFromJSON(doc, jobDefaults); + if (spec.HasMember("parents")) { + const auto &specParents = spec["parents"].GetArray(); + for (size_t c = 0; c < specParents.Size(); ++c) { + task.parents.insert(specParents[c].GetString()); + } } - TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults) { - TaskSet tasks; - if (!spec.IsObject()) { throw std::runtime_error("Tasks is not an object"); } - - // Tasks - for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) { - if (!it->name.IsString()) throw std::runtime_error("Task names must be a string."); - if (!it->value.IsObject()) throw std::runtime_error("Task definitions must be an object."); - const auto &taskName = it->name.GetString(); - tasks.emplace(taskName, taskFromJSON(taskName, it->value, jobDefaults)); + if (spec.HasMember("job")) { + const auto ¶ms = spec["job"]; + if (!params.IsObject()) + throw std::runtime_error("job is not a dictionary."); + for (auto it = params.MemberBegin(); it != params.MemberEnd(); ++it) { + if (!it->name.IsString()) + throw std::runtime_error("job key must be a string."); + if (it->value.IsArray()) { + std::vector values; + for (size_t i = 0; i < it->value.Size(); ++i) { + values.emplace_back(it->value[i].GetString()); + } + task.job.insert_or_assign(it->name.GetString(), values); } - - // Normalize tasks so all the children are populated - for (auto &[k, v] : tasks) { - for (const auto & p : v.parents) { - tasks[p].children.insert(k); - } - v.parents.clear(); + else { + task.job.insert_or_assign(it->name.GetString(), + it->value.GetString()); } - - return tasks; + } } -// I really want to do this with rapidjson, but damn they make it ugly and difficult. -// So we'll shortcut and generate the JSON directly. - std::string taskToJSON(const Task &task) { - std::stringstream ss; - bool first = false; + return task; + } - ss << "{" - << R"("maxRetries": )" << task.maxRetries << ',' - << R"("retryIntervalSeconds": )" << task.retryIntervalSeconds << ','; + TaskSet tasksFromJSON(const std::string &jsonSpec, + const ConfigValues &jobDefaults) + { + rj::Document doc; + checkRJParse(doc.Parse(jsonSpec.c_str())); + return tasksFromJSON(doc, jobDefaults); + } - ss << R"("job": )" << configToJSON(task.job) << ','; - - ss << R"("children": [)"; - first = true; - for (const auto &child: task.children) { - if (!first) ss << ','; - ss << std::quoted(child); - first = false; - } - ss << "],"; - - ss << R"("parents": [)"; - first = true; - for (const auto &parent: task.parents) { - if (!first) ss << ','; - ss << std::quoted(parent); - first = false; - } - ss << "],"; - - ss << R"("isGenerator": )" << (task.isGenerator ? "true" : "false"); - - ss << '}'; - return ss.str(); + TaskSet tasksFromJSON(const rj::Value &spec, const ConfigValues &jobDefaults) + { + TaskSet tasks; + if (!spec.IsObject()) { + throw std::runtime_error("Tasks is not an object"); } - std::string tasksToJSON(const TaskSet &tasks) { - std::stringstream ss; - - ss << "{"; - - bool first = true; - for (const auto &[name, task]: tasks) { - if (!first) ss << ','; - ss << std::quoted(name) << ": " << taskToJSON(task); - first = false; - } - ss << "}"; - - return ss.str(); + // Tasks + for (auto it = spec.MemberBegin(); it != spec.MemberEnd(); ++it) { + if (!it->name.IsString()) + throw std::runtime_error("Task names must be a string."); + if (!it->value.IsObject()) + throw std::runtime_error("Task definitions must be an object."); + const auto &taskName = it->name.GetString(); + tasks.emplace(taskName, taskFromJSON(taskName, it->value, jobDefaults)); } - std::ostream &operator<<(std::ostream &os, const Task &task) { - os << taskToJSON(task); - return os; + // Normalize tasks so all the children are populated + for (auto &[k, v] : tasks) { + for (const auto &p : v.parents) { + tasks[p].children.insert(k); + } + v.parents.clear(); } - std::string attemptRecordToJSON(const AttemptRecord &record) { - std::stringstream ss; + return tasks; + } - ss << "{" - << R"("startTime": )" << std::quoted(timePointToString(record.startTime)) << ',' - << R"("stopTime": )" << std::quoted(timePointToString(record.stopTime)) << ',' - << R"("rc": )" << std::to_string(record.rc) << ',' - << R"("executorLog": )" << std::quoted(record.executorLog) << ',' - << R"("outputLog": )" << std::quoted(record.outputLog) << ',' - << R"("errorLog": )" << std::quoted(record.errorLog) - << '}'; + // I really want to do this with rapidjson, but damn they make it ugly and + // difficult. So we'll shortcut and generate the JSON directly. + std::string taskToJSON(const Task &task) + { + std::stringstream ss; + bool first = false; - return ss.str(); + ss << "{" + << R"("maxRetries": )" << task.maxRetries << ',' + << R"("retryIntervalSeconds": )" << task.retryIntervalSeconds << ','; + + ss << R"("job": )" << configToJSON(task.job) << ','; + + ss << R"("children": [)"; + first = true; + for (const auto &child : task.children) { + if (!first) + ss << ','; + ss << std::quoted(child); + first = false; } + ss << "],"; - std::string timePointToString(const TimePoint &tp) { - std::stringstream ss; - ss << tp; - return ss.str(); + ss << R"("parents": [)"; + first = true; + for (const auto &parent : task.parents) { + if (!first) + ss << ','; + ss << std::quoted(parent); + first = false; } + ss << "],"; - TimePoint stringToTimePoint(const std::string &timeString) { - std::tm dt; - std::stringstream ss{timeString}; - ss >> std::get_time(&dt, "%Y-%m-%d %H:%M:%S %Z"); - return Clock::from_time_t(mktime(&dt)); + ss << R"("isGenerator": )" << (task.isGenerator ? "true" : "false"); + + ss << '}'; + return ss.str(); + } + + std::string tasksToJSON(const TaskSet &tasks) + { + std::stringstream ss; + + ss << "{"; + + bool first = true; + for (const auto &[name, task] : tasks) { + if (!first) + ss << ','; + ss << std::quoted(name) << ": " << taskToJSON(task); + first = false; } + ss << "}"; -} + return ss.str(); + } + + std::ostream &operator<<(std::ostream &os, const Task &task) + { + os << taskToJSON(task); + return os; + } + + std::string attemptRecordToJSON(const AttemptRecord &record) + { + std::stringstream ss; + + ss << "{" + << R"("startTime": )" << std::quoted(timePointToString(record.startTime)) + << ',' << R"("stopTime": )" + << std::quoted(timePointToString(record.stopTime)) << ',' << R"("rc": )" + << std::to_string(record.rc) << ',' << R"("executorLog": )" + << std::quoted(record.executorLog) << ',' << R"("outputLog": )" + << std::quoted(record.outputLog) << ',' << R"("errorLog": )" + << std::quoted(record.errorLog) << '}'; + + return ss.str(); + } + + std::string timePointToString(const TimePoint &tp) + { + std::stringstream ss; + ss << tp; + return ss.str(); + } + + TimePoint stringToTimePoint(const std::string &timeString) + { + std::tm dt; + std::stringstream ss{timeString}; + ss >> std::get_time(&dt, "%Y-%m-%d %H:%M:%S %Z"); + return Clock::from_time_t(mktime(&dt)); + } + +} // namespace daggy diff --git a/daggy/src/Server.cpp b/daggy/src/Server.cpp index 1b5cd33..6049722 100644 --- a/daggy/src/Server.cpp +++ b/daggy/src/Server.cpp @@ -1,260 +1,308 @@ -#include - #include -#include #include +#include #include +#include -#define REQ_ERROR(code, msg) response.send(Pistache::Http::Code::code, msg); return; +#define REQ_ERROR(code, msg) \ + response.send(Pistache::Http::Code::code, msg); \ + return; namespace rj = rapidjson; using namespace Pistache; namespace daggy { - void Server::init(int threads) { - auto opts = Http::Endpoint::options() - .threads(threads) - .flags(Pistache::Tcp::Options::ReuseAddr | Pistache::Tcp::Options::ReusePort) - .maxRequestSize(4294967296) - .maxResponseSize(4294967296); - endpoint_.init(opts); - createDescription(); + void Server::init(int threads) + { + auto opts = Http::Endpoint::options() + .threads(threads) + .flags(Pistache::Tcp::Options::ReuseAddr | + Pistache::Tcp::Options::ReusePort) + .maxRequestSize(4294967296) + .maxResponseSize(4294967296); + endpoint_.init(opts); + createDescription(); + } + + void Server::start() + { + router_.initFromDescription(desc_); + + endpoint_.setHandler(router_.handler()); + endpoint_.serveThreaded(); + } + + void Server::shutdown() + { + endpoint_.shutdown(); + } + + uint16_t Server::getPort() const + { + return endpoint_.getPort(); + } + + void Server::createDescription() + { + desc_.info().license("MIT", "https://opensource.org/licenses/MIT"); + + auto backendErrorResponse = desc_.response( + Http::Code::Internal_Server_Error, "An error occured with the backend"); + + desc_.schemes(Rest::Scheme::Http) + .basePath("/v1") + .produces(MIME(Application, Json)) + .consumes(MIME(Application, Json)); + + desc_.route(desc_.get("/ready")) + .bind(&Server::handleReady, this) + .response(Http::Code::Ok, "Response to the /ready call") + .hide(); + + auto versionPath = desc_.path("/v1"); + + auto dagPath = versionPath.path("/dagrun"); + + // Run a DAG + dagPath.route(desc_.post("/")) + .bind(&Server::handleRunDAG, this) + .produces(MIME(Application, Json), MIME(Application, Xml)) + .response(Http::Code::Ok, "Run a DAG"); + // List detailed DAG run + dagPath.route(desc_.get("/:runID")) + .bind(&Server::handleGetDAGRun, this) + .produces(MIME(Application, Json), MIME(Application, Xml)) + .response(Http::Code::Ok, "Details of a specific DAG run"); + + // List all DAG runs + dagPath.route(desc_.get("/")) + .bind(&Server::handleGetDAGRuns, this) + .produces(MIME(Application, Json), MIME(Application, Xml)) + .response(Http::Code::Ok, "The list of all known DAG Runs"); + } + + /* + * { + * "name": "DAG Run Name" + * "job": {...} + * "tasks": {...} + */ + void Server::handleRunDAG(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response) + { + if (!handleAuth(request, response)) + return; + + rj::Document doc; + try { + doc.Parse(request.body().c_str()); + } + catch (std::exception &e) { + REQ_ERROR(Bad_Request, std::string{"Invalid JSON payload: "} + e.what()); } - void Server::start() { - router_.initFromDescription(desc_); - - endpoint_.setHandler(router_.handler()); - endpoint_.serveThreaded(); + if (!doc.IsObject()) { + REQ_ERROR(Bad_Request, "Payload is not a dictionary."); + } + if (!doc.HasMember("name")) { + REQ_ERROR(Bad_Request, "DAG Run is missing a name."); + } + if (!doc.HasMember("tasks")) { + REQ_ERROR(Bad_Request, "DAG Run has no tasks."); } - void Server::shutdown() { - endpoint_.shutdown(); + std::string runName = doc["name"].GetString(); + + // Get parameters if there are any + ConfigValues parameters; + if (doc.HasMember("parameters")) { + try { + auto parsedParams = configFromJSON(doc["parameters"].GetObject()); + parameters.swap(parsedParams); + } + catch (std::exception &e) { + REQ_ERROR(Bad_Request, e.what()); + } } - uint16_t Server::getPort() const { - return endpoint_.getPort(); + // Job Defaults + ConfigValues jobDefaults; + if (doc.HasMember("jobDefaults")) { + try { + auto parsedJobDefaults = configFromJSON(doc["jobDefaults"].GetObject()); + jobDefaults.swap(parsedJobDefaults); + } + catch (std::exception &e) { + REQ_ERROR(Bad_Request, e.what()); + } } - void Server::createDescription() { - desc_ - .info() - .license("MIT", "https://opensource.org/licenses/MIT"); - - - auto backendErrorResponse = desc_.response(Http::Code::Internal_Server_Error, - "An error occured with the backend"); - - desc_ - .schemes(Rest::Scheme::Http) - .basePath("/v1") - .produces(MIME(Application, Json)) - .consumes(MIME(Application, Json)); - - desc_ - .route(desc_.get("/ready")) - .bind(&Server::handleReady, this) - .response(Http::Code::Ok, "Response to the /ready call") - .hide(); - - - auto versionPath = desc_.path("/v1"); - - auto dagPath = versionPath.path("/dagrun"); - - // Run a DAG - dagPath - .route(desc_.post("/")) - .bind(&Server::handleRunDAG, this) - .produces(MIME(Application, Json), MIME(Application, Xml)) - .response(Http::Code::Ok, "Run a DAG"); - // List detailed DAG run - dagPath - .route(desc_.get("/:runID")) - .bind(&Server::handleGetDAGRun, this) - .produces(MIME(Application, Json), MIME(Application, Xml)) - .response(Http::Code::Ok, "Details of a specific DAG run"); - - // List all DAG runs - dagPath - .route(desc_.get("/")) - .bind(&Server::handleGetDAGRuns, this) - .produces(MIME(Application, Json), MIME(Application, Xml)) - .response(Http::Code::Ok, "The list of all known DAG Runs"); + // Get the tasks + TaskSet tasks; + try { + auto taskTemplates = tasksFromJSON(doc["tasks"], jobDefaults); + auto expandedTasks = expandTaskSet(taskTemplates, executor_, parameters); + tasks.swap(expandedTasks); + } + catch (std::exception &e) { + REQ_ERROR(Bad_Request, e.what()); } - /* - * { - * "name": "DAG Run Name" - * "job": {...} - * "tasks": {...} - */ - void Server::handleRunDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { - if (!handleAuth(request, response)) return; + // Get a run ID + auto runID = logger_.startDAGRun(runName, tasks); + auto dag = buildDAGFromTasks(tasks); - rj::Document doc; - try { - doc.Parse(request.body().c_str()); - } catch (std::exception &e) { - REQ_ERROR(Bad_Request, std::string{"Invalid JSON payload: "} + e.what()); + runnerPool_.addTask([this, parameters, runID, dag]() { + runDAG(runID, this->executor_, this->logger_, dag, parameters); + }); + + response.send(Pistache::Http::Code::Ok, + R"({"runID": )" + std::to_string(runID) + "}"); + } + + void Server::handleGetDAGRuns(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response) + { + if (!handleAuth(request, response)) + return; + auto dagRuns = logger_.getDAGs(0); + std::stringstream ss; + ss << '['; + + bool first = true; + for (const auto &run : dagRuns) { + if (first) { + first = false; + } + else { + ss << ", "; + } + + ss << " {" + << R"("runID": )" << run.runID << ',' << R"("name": )" + << std::quoted(run.name) << "," + << R"("startTime": )" << std::quoted(timePointToString(run.startTime)) + << ',' << R"("lastUpdate": )" + << std::quoted(timePointToString(run.lastUpdate)) << ',' + << R"("taskCounts": {)"; + bool firstState = true; + for (const auto &[state, count] : run.taskStateCounts) { + if (firstState) { + firstState = false; } - - if (!doc.IsObject()) { REQ_ERROR(Bad_Request, "Payload is not a dictionary."); } - if (!doc.HasMember("name")) { REQ_ERROR(Bad_Request, "DAG Run is missing a name."); } - if (!doc.HasMember("tasks")) { REQ_ERROR(Bad_Request, "DAG Run has no tasks."); } - - std::string runName = doc["name"].GetString(); - - // Get parameters if there are any - ConfigValues parameters; - if (doc.HasMember("parameters")) { - try { - auto parsedParams = configFromJSON(doc["parameters"].GetObject()); - parameters.swap(parsedParams); - } catch (std::exception &e) { - REQ_ERROR(Bad_Request, e.what()); - } + else { + ss << ", "; } - - // Job Defaults - ConfigValues jobDefaults; - if (doc.HasMember("jobDefaults")) { - try { - auto parsedJobDefaults = configFromJSON(doc["jobDefaults"].GetObject()); - jobDefaults.swap(parsedJobDefaults); - } catch (std::exception &e) { - REQ_ERROR(Bad_Request, e.what()); - } - } - - // Get the tasks - TaskSet tasks; - try { - auto taskTemplates = tasksFromJSON(doc["tasks"], jobDefaults); - auto expandedTasks = expandTaskSet(taskTemplates, executor_, parameters); - tasks.swap(expandedTasks); - } catch (std::exception &e) { - REQ_ERROR(Bad_Request, e.what()); - } - - // Get a run ID - auto runID = logger_.startDAGRun(runName, tasks); - auto dag = buildDAGFromTasks(tasks); - - runnerPool_.addTask( - [this, parameters, runID, dag]() { runDAG(runID, this->executor_, this->logger_, dag, parameters); }); - - response.send(Pistache::Http::Code::Ok, R"({"runID": )" + std::to_string(runID) + "}"); + ss << std::quoted(state._to_string()) << ':' << count; + } + ss << '}' // end of taskCounts + << '}'; // end of item } - void Server::handleGetDAGRuns(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { - if (!handleAuth(request, response)) return; - auto dagRuns = logger_.getDAGs(0); - std::stringstream ss; - ss << '['; + ss << ']'; + response.send(Pistache::Http::Code::Ok, ss.str()); + } - bool first = true; - for (const auto &run: dagRuns) { - if (first) { - first = false; - } else { - ss << ", "; - } - - ss << " {" - << R"("runID": )" << run.runID << ',' - << R"("name": )" << std::quoted(run.name) << "," - << R"("startTime": )" << std::quoted(timePointToString(run.startTime)) << ',' - << R"("lastUpdate": )" << std::quoted(timePointToString(run.lastUpdate)) << ',' - << R"("taskCounts": {)"; - bool firstState = true; - for (const auto &[state, count]: run.taskStateCounts) { - if (firstState) { - firstState = false; - } else { - ss << ", "; - } - ss << std::quoted(state._to_string()) << ':' << count; - } - ss << '}' // end of taskCounts - << '}'; // end of item - } - - ss << ']'; - response.send(Pistache::Http::Code::Ok, ss.str()); + void Server::handleGetDAGRun(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response) + { + if (!handleAuth(request, response)) + return; + if (!request.hasParam(":runID")) { + REQ_ERROR(Not_Found, "No runID provided in URL"); } + DAGRunID runID = request.param(":runID").as(); + auto run = logger_.getDAGRun(runID); - void Server::handleGetDAGRun(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { - if (!handleAuth(request, response)) return; - if (!request.hasParam(":runID")) { REQ_ERROR(Not_Found, "No runID provided in URL"); } - DAGRunID runID = request.param(":runID").as(); - auto run = logger_.getDAGRun(runID); + bool first = true; + std::stringstream ss; + ss << "{" + << R"("runID": )" << runID << ',' << R"("name": )" + << std::quoted(run.name) << ',' << R"("tasks": )" + << tasksToJSON(run.tasks) << ','; - bool first = true; - std::stringstream ss; - ss << "{" - << R"("runID": )" << runID << ',' - << R"("name": )" << std::quoted(run.name) << ',' - << R"("tasks": )" << tasksToJSON(run.tasks) << ','; - - // task run states - ss << R"("taskStates": { )"; - first = true; - for (const auto &[name, state]: run.taskRunStates) { - if (first) { first = false; } else { ss << ','; } - ss << std::quoted(name) << ": " << std::quoted(state._to_string()); - } - ss << "},"; - - // Attempt records - first = true; - ss << R"("taskAttempts": { )"; - for (const auto &[taskName, attempts]: run.taskAttempts) { - if (first) { first = false; } else { ss << ','; } - ss << std::quoted(taskName) << ": ["; - bool firstAttempt = true; - for (const auto &attempt: attempts) { - if (firstAttempt) { firstAttempt = false; } else { ss << ','; } - ss << '{' - << R"("startTime":)" << std::quoted(timePointToString(attempt.startTime)) << ',' - << R"("stopTime":)" << std::quoted(timePointToString(attempt.stopTime)) << ',' - << R"("rc":)" << attempt.rc << ',' - << R"("outputLog":)" << std::quoted(attempt.outputLog) << ',' - << R"("errorLog":)" << std::quoted(attempt.errorLog) << ',' - << R"("executorLog":)" << std::quoted(attempt.executorLog) - << '}'; - } - ss << ']'; - } - ss << "},"; - - // DAG state changes - first = true; - ss << R"("dagStateChanges": [ )"; - for (const auto &change: run.dagStateChanges) { - if (first) { first = false; } else { ss << ','; } - ss << '{' - << R"("newState": )" << std::quoted(change.newState._to_string()) << ',' - << R"("time": )" << std::quoted(timePointToString(change.time)) - << '}'; - } - ss << "]"; - ss << '}'; - - response.send(Pistache::Http::Code::Ok, ss.str()); + // task run states + ss << R"("taskStates": { )"; + first = true; + for (const auto &[name, state] : run.taskRunStates) { + if (first) { + first = false; + } + else { + ss << ','; + } + ss << std::quoted(name) << ": " << std::quoted(state._to_string()); } + ss << "},"; - void Server::handleReady(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response) { - response.send(Pistache::Http::Code::Ok, "Ya like DAGs?"); + // Attempt records + first = true; + ss << R"("taskAttempts": { )"; + for (const auto &[taskName, attempts] : run.taskAttempts) { + if (first) { + first = false; + } + else { + ss << ','; + } + ss << std::quoted(taskName) << ": ["; + bool firstAttempt = true; + for (const auto &attempt : attempts) { + if (firstAttempt) { + firstAttempt = false; + } + else { + ss << ','; + } + ss << '{' << R"("startTime":)" + << std::quoted(timePointToString(attempt.startTime)) << ',' + << R"("stopTime":)" + << std::quoted(timePointToString(attempt.stopTime)) << ',' + << R"("rc":)" << attempt.rc << ',' << R"("outputLog":)" + << std::quoted(attempt.outputLog) << ',' << R"("errorLog":)" + << std::quoted(attempt.errorLog) << ',' << R"("executorLog":)" + << std::quoted(attempt.executorLog) << '}'; + } + ss << ']'; } + ss << "},"; - /* - * handleAuth will check any auth methods and handle any responses in the case of failed auth. If it returns - * false, callers should cease handling the response - */ - bool Server::handleAuth(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter &response) { - (void) response; - return true; + // DAG state changes + first = true; + ss << R"("dagStateChanges": [ )"; + for (const auto &change : run.dagStateChanges) { + if (first) { + first = false; + } + else { + ss << ','; + } + ss << '{' << R"("newState": )" + << std::quoted(change.newState._to_string()) << ',' << R"("time": )" + << std::quoted(timePointToString(change.time)) << '}'; } -} + ss << "]"; + ss << '}'; + + response.send(Pistache::Http::Code::Ok, ss.str()); + } + + void Server::handleReady(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter response) + { + response.send(Pistache::Http::Code::Ok, "Ya like DAGs?"); + } + + /* + * handleAuth will check any auth methods and handle any responses in the case + * of failed auth. If it returns false, callers should cease handling the + * response + */ + bool Server::handleAuth(const Pistache::Rest::Request &request, + Pistache::Http::ResponseWriter &response) + { + (void)response; + return true; + } +} // namespace daggy diff --git a/daggy/src/Utilities.cpp b/daggy/src/Utilities.cpp index be4bddd..48cf17c 100644 --- a/daggy/src/Utilities.cpp +++ b/daggy/src/Utilities.cpp @@ -1,219 +1,234 @@ +#include +#include #include #include -#include -#include - using namespace std::chrono_literals; namespace daggy { - std::string globalSub(std::string string, const std::string &pattern, const std::string &replacement) { - size_t pos = string.find(pattern); - while (pos != std::string::npos) { - string.replace(pos, pattern.size(), replacement); - pos = string.find(pattern); + std::string globalSub(std::string string, const std::string &pattern, + const std::string &replacement) + { + size_t pos = string.find(pattern); + while (pos != std::string::npos) { + string.replace(pos, pattern.size(), replacement); + pos = string.find(pattern); + } + return string; + } + + std::vector> interpolateValues( + const std::vector &raw, const ConfigValues &values) + { + std::vector> cooked{{}}; + + for (const auto &part : raw) { + std::vector expandedPart{part}; + + // Find all values of parameters, and expand them + for (const auto &[paramRaw, paramValue] : values) { + std::string param = "{{" + paramRaw + "}}"; + auto pos = part.find(param); + if (pos == std::string::npos) + continue; + std::vector newExpandedPart; + + if (std::holds_alternative(paramValue)) { + for (auto &cmd : expandedPart) { + newExpandedPart.push_back( + globalSub(cmd, param, std::get(paramValue))); + } } - return string; + else { + for (const auto &val : + std::get>(paramValue)) { + for (auto cmd : expandedPart) { + newExpandedPart.push_back(globalSub(cmd, param, val)); + } + } + } + + expandedPart.swap(newExpandedPart); + } + + std::vector> newCommands; + for (const auto &newPart : expandedPart) { + for (auto cmd : cooked) { + cmd.push_back(newPart); + newCommands.emplace_back(cmd); + } + } + cooked.swap(newCommands); + } + return cooked; + } + + TaskSet expandTaskSet(const TaskSet &tasks, + executors::task::TaskExecutor &executor, + const ConfigValues &interpolatedValues) + { + // Expand the tasks first + TaskSet newTaskSet; + for (const auto &[baseName, task] : tasks) { + executor.validateTaskParameters(task.job); + const auto newJobs = + executor.expandTaskParameters(task.job, interpolatedValues); + size_t i = 0; + for (const auto &newJob : newJobs) { + Task newTask{task}; + newTask.job = newJob; + newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask); + ++i; + } + } + return newTaskSet; + } + + void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks) + { + // Add the missing vertices + for (const auto &[name, task] : tasks) { + dag.addVertex(name, task); } - std::vector> - interpolateValues(const std::vector &raw, const ConfigValues &values) { - std::vector> cooked{{}}; + // Add edges + for (const auto &[name, task] : tasks) { + dag.addEdgeIf(name, [&](const auto &v) { + return task.children.count(v.data.definedName) > 0; + }); + } - for (const auto &part: raw) { - std::vector expandedPart{part}; + if (!dag.isValid()) { + throw std::runtime_error("DAG contains a cycle"); + } + } - // Find all values of parameters, and expand them - for (const auto &[paramRaw, paramValue]: values) { - std::string param = "{{" + paramRaw + "}}"; - auto pos = part.find(param); - if (pos == std::string::npos) continue; - std::vector newExpandedPart; + TaskDAG buildDAGFromTasks( + TaskSet &tasks, + const std::vector &updates) + { + TaskDAG dag; + updateDAGFromTasks(dag, tasks); - if (std::holds_alternative(paramValue)) { - for (auto &cmd: expandedPart) { - newExpandedPart.push_back(globalSub(cmd, param, std::get(paramValue))); - } - } else { - for (const auto &val: std::get>(paramValue)) { - for (auto cmd: expandedPart) { - newExpandedPart.push_back(globalSub(cmd, param, val)); - } - } + // Replay any updates + for (const auto &update : updates) { + switch (update.newState) { + case RunState::RUNNING: + case RunState::RETRY: + case RunState::ERRORED: + case RunState::KILLED: + dag.setVertexState(update.taskName, RunState::RUNNING); + dag.setVertexState(update.taskName, RunState::COMPLETED); + break; + case RunState::COMPLETED: + case RunState::QUEUED: + break; + } + } + + return dag; + } + + TaskDAG runDAG(DAGRunID runID, executors::task::TaskExecutor &executor, + loggers::dag_run::DAGRunLogger &logger, TaskDAG dag, + const ConfigValues parameters) + { + logger.updateDAGRunState(runID, RunState::RUNNING); + + std::unordered_map> runningTasks; + std::unordered_map taskAttemptCounts; + + size_t running = 0; + size_t errored = 0; + while (!dag.allVisited()) { + // Check for any completed tasks + for (auto &[taskName, fut] : runningTasks) { + if (fut.valid()) { + auto attempt = fut.get(); + logger.logTaskAttempt(runID, taskName, attempt); + auto &vert = dag.getVertex(taskName); + auto &task = vert.data; + if (attempt.rc == 0) { + logger.updateTaskState(runID, taskName, RunState::COMPLETED); + if (task.isGenerator) { + // Parse the output and update the DAGs + try { + auto newTasks = expandTaskSet(tasksFromJSON(attempt.outputLog), + executor, parameters); + updateDAGFromTasks(dag, newTasks); + + for (const auto &[ntName, ntTask] : newTasks) { + logger.addTask(runID, ntName, ntTask); + dag.addEdge(taskName, ntName); + task.children.insert(ntName); } - - expandedPart.swap(newExpandedPart); + logger.updateTask(runID, taskName, task); + } + catch (std::exception &e) { + logger.logTaskAttempt( + runID, taskName, + AttemptRecord{ + .executorLog = + std::string{"Failed to parse JSON output: "} + + e.what()}); + logger.updateTaskState(runID, taskName, RunState::ERRORED); + ++errored; + } } - - std::vector> newCommands; - for (const auto &newPart: expandedPart) { - for (auto cmd: cooked) { - cmd.push_back(newPart); - newCommands.emplace_back(cmd); - } + dag.completeVisit(taskName); + --running; + } + else { + // RC isn't 0 + if (taskAttemptCounts[taskName] <= task.maxRetries) { + logger.updateTaskState(runID, taskName, RunState::RETRY); + runningTasks[taskName] = executor.execute(taskName, task); + ++taskAttemptCounts[taskName]; } - cooked.swap(newCommands); + else { + logger.updateTaskState(runID, taskName, RunState::ERRORED); + ++errored; + } + } } - return cooked; + } + + // Add all remaining tasks in a task queue to avoid dominating the thread + // pool + auto t = dag.visitNext(); + while (t.has_value()) { + // Schedule the task to run + auto &taskName = t.value().first; + auto &task = t.value().second; + taskAttemptCounts[taskName] = 1; + + logger.updateTaskState(runID, taskName, RunState::RUNNING); + runningTasks.emplace(taskName, executor.execute(taskName, task)); + ++running; + + auto nextTask = dag.visitNext(); + if (not nextTask.has_value()) + break; + t.emplace(nextTask.value()); + } + if (running > 0 and errored == running) { + logger.updateDAGRunState(runID, RunState::ERRORED); + break; + } + std::this_thread::sleep_for(250ms); } - TaskSet - expandTaskSet(const TaskSet &tasks, - executors::task::TaskExecutor &executor, - const ConfigValues &interpolatedValues) { - // Expand the tasks first - TaskSet newTaskSet; - for (const auto &[baseName, task]: tasks) { - executor.validateTaskParameters(task.job); - const auto newJobs = executor.expandTaskParameters(task.job, interpolatedValues); - size_t i = 0; - for (const auto &newJob: newJobs) { - Task newTask{task}; - newTask.job = newJob; - newTaskSet.emplace(baseName + "_" + std::to_string(i), newTask); - ++i; - } - } - return newTaskSet; + if (dag.allVisited()) { + logger.updateDAGRunState(runID, RunState::COMPLETED); } + return dag; + } - void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks) { - // Add the missing vertices - for (const auto &[name, task]: tasks) { - dag.addVertex(name, task); - } - - // Add edges - for (const auto &[name, task]: tasks) { - dag.addEdgeIf(name, [&](const auto &v) { return task.children.count(v.data.definedName) > 0; }); - } - - if (! dag.isValid()) { - throw std::runtime_error("DAG contains a cycle"); - } - } - - TaskDAG buildDAGFromTasks(TaskSet &tasks, - const std::vector &updates) { - TaskDAG dag; - updateDAGFromTasks(dag, tasks); - - // Replay any updates - for (const auto &update: updates) { - switch (update.newState) { - case RunState::RUNNING: - case RunState::RETRY: - case RunState::ERRORED: - case RunState::KILLED: - dag.setVertexState(update.taskName, RunState::RUNNING); - dag.setVertexState(update.taskName, RunState::COMPLETED); - break; - case RunState::COMPLETED: - case RunState::QUEUED: - break; - } - } - - return dag; - } - - TaskDAG runDAG(DAGRunID runID, - executors::task::TaskExecutor &executor, - loggers::dag_run::DAGRunLogger &logger, - TaskDAG dag, - const ConfigValues parameters - ) { - logger.updateDAGRunState(runID, RunState::RUNNING); - - std::unordered_map> runningTasks; - std::unordered_map taskAttemptCounts; - - size_t running = 0; - size_t errored = 0; - while (!dag.allVisited()) { - // Check for any completed tasks - for (auto &[taskName, fut]: runningTasks) { - if (fut.valid()) { - auto attempt = fut.get(); - logger.logTaskAttempt(runID, taskName, attempt); - auto &vert = dag.getVertex(taskName); - auto &task = vert.data; - if (attempt.rc == 0) { - logger.updateTaskState(runID, taskName, RunState::COMPLETED); - if (task.isGenerator) { - // Parse the output and update the DAGs - try { - auto newTasks = expandTaskSet(tasksFromJSON(attempt.outputLog), - executor, - parameters - ); - updateDAGFromTasks(dag, newTasks); - - for (const auto &[ntName, ntTask]: newTasks) { - logger.addTask(runID, ntName, ntTask); - dag.addEdge(taskName, ntName); - task.children.insert(ntName); - } - logger.updateTask(runID, taskName, task); - } catch (std::exception &e) { - logger.logTaskAttempt(runID, taskName, - AttemptRecord{.executorLog = - std::string{"Failed to parse JSON output: "} + - e.what()}); - logger.updateTaskState(runID, taskName, RunState::ERRORED); - ++errored; - } - } - dag.completeVisit(taskName); - --running; - } else { - // RC isn't 0 - if (taskAttemptCounts[taskName] <= task.maxRetries) { - logger.updateTaskState(runID, taskName, RunState::RETRY); - runningTasks[taskName] = executor.execute(taskName, task); - ++taskAttemptCounts[taskName]; - } else { - logger.updateTaskState(runID, taskName, RunState::ERRORED); - ++errored; - } - } - } - } - - // Add all remaining tasks in a task queue to avoid dominating the thread pool - auto t = dag.visitNext(); - while (t.has_value()) { - // Schedule the task to run - auto &taskName = t.value().first; - auto &task = t.value().second; - taskAttemptCounts[taskName] = 1; - - logger.updateTaskState(runID, taskName, RunState::RUNNING); - runningTasks.emplace(taskName, executor.execute(taskName, task)); - ++running; - - auto nextTask = dag.visitNext(); - if (not nextTask.has_value()) break; - t.emplace(nextTask.value()); - } - if (running > 0 and errored == running) { - logger.updateDAGRunState(runID, RunState::ERRORED); - break; - } - std::this_thread::sleep_for(250ms); - } - - if (dag.allVisited()) { - logger.updateDAGRunState(runID, RunState::COMPLETED); - } - - return dag; - } - - std::ostream &operator<<(std::ostream &os, const TimePoint &tp) { - auto t_c = Clock::to_time_t(tp); - os << std::put_time(std::localtime(&t_c), "%Y-%m-%d %H:%M:%S %Z"); - return os; - } -} + std::ostream &operator<<(std::ostream &os, const TimePoint &tp) + { + auto t_c = Clock::to_time_t(tp); + os << std::put_time(std::localtime(&t_c), "%Y-%m-%d %H:%M:%S %Z"); + return os; + } +} // namespace daggy diff --git a/daggy/src/executors/task/ForkingTaskExecutor.cpp b/daggy/src/executors/task/ForkingTaskExecutor.cpp index 4c4bf01..a3df342 100644 --- a/daggy/src/executors/task/ForkingTaskExecutor.cpp +++ b/daggy/src/executors/task/ForkingTaskExecutor.cpp @@ -1,122 +1,140 @@ -#include -#include - #include +#include #include #include -#include + +#include +#include using namespace daggy::executors::task; -std::string slurp(int fd) { - std::string result; +std::string slurp(int fd) +{ + std::string result; - const ssize_t BUFFER_SIZE = 4096; - char buffer[BUFFER_SIZE]; + const ssize_t BUFFER_SIZE = 4096; + char buffer[BUFFER_SIZE]; - struct pollfd pfd{.fd = fd, .events = POLLIN, .revents = 0}; + struct pollfd pfd + { + .fd = fd, .events = POLLIN, .revents = 0 + }; + poll(&pfd, 1, 1); + + while (pfd.revents & POLLIN) { + ssize_t bytes = read(fd, buffer, BUFFER_SIZE); + if (bytes == 0) { + break; + } + else { + result.append(buffer, bytes); + } + pfd.revents = 0; poll(&pfd, 1, 1); + } - while (pfd.revents & POLLIN) { - ssize_t bytes = read(fd, buffer, BUFFER_SIZE); - if (bytes == 0) { - break; - } else { - result.append(buffer, bytes); - } - pfd.revents = 0; - poll(&pfd, 1, 1); - } - - return result; + return result; } - -std::future -ForkingTaskExecutor::execute(const std::string &taskName, const Task &task) { - return tp_.addTask([this, task](){return this->runTask(task);}); +std::future ForkingTaskExecutor::execute( + const std::string &taskName, const Task &task) +{ + return tp_.addTask([this, task]() { return this->runTask(task); }); } -daggy::AttemptRecord -ForkingTaskExecutor::runTask(const Task & task) { - AttemptRecord rec; +daggy::AttemptRecord ForkingTaskExecutor::runTask(const Task &task) +{ + AttemptRecord rec; - rec.startTime = Clock::now(); + rec.startTime = Clock::now(); - // Need to convert the strings - std::vector argv; - const auto command = std::get(task.job.at("command")); - std::transform(command.begin(), - command.end(), - std::back_inserter(argv), - [](const std::string & s) { - return const_cast(s.c_str()); - }); - argv.push_back(nullptr); + // Need to convert the strings + std::vector argv; + const auto command = std::get(task.job.at("command")); + std::transform( + command.begin(), command.end(), std::back_inserter(argv), + [](const std::string &s) { return const_cast(s.c_str()); }); + argv.push_back(nullptr); - // Create the pipe - int stdoutPipe[2]; - int pipeRC = pipe2(stdoutPipe, O_DIRECT); - if (pipeRC != 0) throw std::runtime_error("Unable to create pipe for stdout"); - int stderrPipe[2]; - pipeRC = pipe2(stderrPipe, O_DIRECT); - if (pipeRC != 0) throw std::runtime_error("Unable to create pipe for stderr"); + // Create the pipe + int stdoutPipe[2]; + int pipeRC = pipe2(stdoutPipe, O_DIRECT); + if (pipeRC != 0) + throw std::runtime_error("Unable to create pipe for stdout"); + int stderrPipe[2]; + pipeRC = pipe2(stderrPipe, O_DIRECT); + if (pipeRC != 0) + throw std::runtime_error("Unable to create pipe for stderr"); - pid_t child = fork(); - if (child < 0) { - throw std::runtime_error("Unable to fork child"); - } else if (child == 0) { // child - while ((dup2(stdoutPipe[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {} - while ((dup2(stderrPipe[1], STDERR_FILENO) == -1) && (errno == EINTR)) {} - close(stdoutPipe[0]); - close(stderrPipe[0]); - execvp(argv[0], argv.data()); - exit(-1); + pid_t child = fork(); + if (child < 0) { + throw std::runtime_error("Unable to fork child"); + } + else if (child == 0) { // child + while ((dup2(stdoutPipe[1], STDOUT_FILENO) == -1) && (errno == EINTR)) { } - - std::atomic running = true; - std::thread stdoutReader([&]() { while (running) rec.outputLog.append(slurp(stdoutPipe[0])); }); - std::thread stderrReader([&]() { while (running) rec.errorLog.append(slurp(stderrPipe[0])); }); - - int rc = 0; - waitpid(child, &rc, 0); - running = false; - - rec.stopTime = Clock::now(); - if (WIFEXITED(rc)) { - rec.rc = WEXITSTATUS(rc); - } else { - rec.rc = -1; + while ((dup2(stderrPipe[1], STDERR_FILENO) == -1) && (errno == EINTR)) { } - - stdoutReader.join(); - stderrReader.join(); - close(stdoutPipe[0]); close(stderrPipe[0]); + execvp(argv[0], argv.data()); + exit(-1); + } - return rec; + std::atomic running = true; + std::thread stdoutReader([&]() { + while (running) + rec.outputLog.append(slurp(stdoutPipe[0])); + }); + std::thread stderrReader([&]() { + while (running) + rec.errorLog.append(slurp(stderrPipe[0])); + }); + + int rc = 0; + waitpid(child, &rc, 0); + running = false; + + rec.stopTime = Clock::now(); + if (WIFEXITED(rc)) { + rec.rc = WEXITSTATUS(rc); + } + else { + rec.rc = -1; + } + + stdoutReader.join(); + stderrReader.join(); + + close(stdoutPipe[0]); + close(stderrPipe[0]); + + return rec; } -bool ForkingTaskExecutor::validateTaskParameters(const ConfigValues &job) { - auto it = job.find("command"); - if (it == job.end()) - throw std::runtime_error(R"(job does not have a "command" argument)"); - if (!std::holds_alternative(it->second)) - throw std::runtime_error(R"(taskParameter's "command" must be an array of strings)"); - return true; +bool ForkingTaskExecutor::validateTaskParameters(const ConfigValues &job) +{ + auto it = job.find("command"); + if (it == job.end()) + throw std::runtime_error(R"(job does not have a "command" argument)"); + if (!std::holds_alternative(it->second)) + throw std::runtime_error( + R"(taskParameter's "command" must be an array of strings)"); + return true; } -std::vector -ForkingTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) { - std::vector newValues; +std::vector ForkingTaskExecutor::expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) +{ + std::vector newValues; - const auto command = std::get(job.at("command")); - for (const auto &expandedCommand: interpolateValues(command, expansionValues)) { - ConfigValues newCommand{job}; - newCommand.at("command") = expandedCommand; - newValues.emplace_back(newCommand); - } + const auto command = std::get(job.at("command")); + for (const auto &expandedCommand : + interpolateValues(command, expansionValues)) { + ConfigValues newCommand{job}; + newCommand.at("command") = expandedCommand; + newValues.emplace_back(newCommand); + } - return newValues; + return newValues; } diff --git a/daggy/src/executors/task/NoopTaskExecutor.cpp b/daggy/src/executors/task/NoopTaskExecutor.cpp index c56f0ee..9239b8b 100644 --- a/daggy/src/executors/task/NoopTaskExecutor.cpp +++ b/daggy/src/executors/task/NoopTaskExecutor.cpp @@ -1,42 +1,45 @@ -#include #include +#include namespace daggy::executors::task { - std::future - NoopTaskExecutor::execute(const std::string &taskName, const Task &task) { - std::promise promise; - auto ts = Clock::now(); - promise.set_value(AttemptRecord{ - .startTime = ts, - .stopTime = ts, - .rc = 0, - .executorLog = taskName, - .outputLog = taskName, - .errorLog = taskName - }); - return promise.get_future(); - } + std::future NoopTaskExecutor::execute( + const std::string &taskName, const Task &task) + { + std::promise promise; + auto ts = Clock::now(); + promise.set_value(AttemptRecord{.startTime = ts, + .stopTime = ts, + .rc = 0, + .executorLog = taskName, + .outputLog = taskName, + .errorLog = taskName}); + return promise.get_future(); + } - bool NoopTaskExecutor::validateTaskParameters(const ConfigValues &job) { - auto it = job.find("command"); - if (it == job.end()) - throw std::runtime_error(R"(job does not have a "command" argument)"); - if (!std::holds_alternative(it->second)) - throw std::runtime_error(R"(taskParameter's "command" must be an array of strings)"); - return true; + bool NoopTaskExecutor::validateTaskParameters(const ConfigValues &job) + { + auto it = job.find("command"); + if (it == job.end()) + throw std::runtime_error(R"(job does not have a "command" argument)"); + if (!std::holds_alternative(it->second)) + throw std::runtime_error( + R"(taskParameter's "command" must be an array of strings)"); + return true; + } + + std::vector NoopTaskExecutor::expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) + { + std::vector newValues; + + const auto command = std::get(job.at("command")); + for (const auto &expandedCommand : + interpolateValues(command, expansionValues)) { + ConfigValues newCommand{job}; + newCommand.at("command") = expandedCommand; + newValues.emplace_back(newCommand); } - std::vector - NoopTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) { - std::vector newValues; - - const auto command = std::get(job.at("command")); - for (const auto &expandedCommand: interpolateValues(command, expansionValues)) { - ConfigValues newCommand{job}; - newCommand.at("command") = expandedCommand; - newValues.emplace_back(newCommand); - } - - return newValues; - } -} + return newValues; + } +} // namespace daggy::executors::task diff --git a/daggy/src/executors/task/SlurmTaskExecutor.cpp b/daggy/src/executors/task/SlurmTaskExecutor.cpp index c9914aa..5c2cc38 100644 --- a/daggy/src/executors/task/SlurmTaskExecutor.cpp +++ b/daggy/src/executors/task/SlurmTaskExecutor.cpp @@ -1,274 +1,274 @@ #include #include #ifdef DAGGY_ENABLE_SLURM -#include +#include +#include +#include +#include +#include +#include + +#include +#include #include #include - -#include -#include -#include -#include -#include - -#include - -#include -#include +#include namespace fs = std::filesystem; namespace daggy::executors::task { - std::string getUniqueTag(size_t nChars = 6) { - std::string result(nChars, '\0'); - static std::random_device dev; - static std::mt19937 rng(dev()); + std::string getUniqueTag(size_t nChars = 6) + { + std::string result(nChars, '\0'); + static std::random_device dev; + static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(0, 61); + std::uniform_int_distribution dist(0, 61); - const char *v = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const char *v = + "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; - for (size_t i = 0; i < nChars; i++) { - result[i] = v[dist(rng)]; - } - return result; + for (size_t i = 0; i < nChars; i++) { + result[i] = v[dist(rng)]; + } + return result; + } + + void readAndClean(const fs::path &fn, std::string &dest) + { + if (!fs::exists(fn)) + return; + + std::ifstream ifh; + ifh.open(fn); + std::string contents(std::istreambuf_iterator{ifh}, {}); + ifh.close(); + fs::remove_all(fn); + + dest.swap(contents); + } + + SlurmTaskExecutor::SlurmTaskExecutor() + : running_(true) + , monitorWorker_(&SlurmTaskExecutor::monitor, this) + { + std::string priority = + "SLURM_PRIO_PROCESS=" + std::to_string(getpriority(PRIO_PROCESS, 0)); + std::string submitDir = "SLURM_SUBMIT_DIR=" + fs::current_path().string(); + + const size_t MAX_HOSTNAME_LENGTH = 50; + std::string submitHost(MAX_HOSTNAME_LENGTH, '\0'); + gethostname(submitHost.data(), MAX_HOSTNAME_LENGTH); + submitHost = "SLURM_SUBMIT_HOST=" + submitHost; + submitHost.resize(submitHost.find('\0')); + + uint32_t mask = umask(0); + umask(mask); // Restore the old mask + + std::stringstream ss; + ss << "SLURM_UMASK=0" << uint32_t{((mask >> 6) & 07)} + << uint32_t{((mask >> 3) & 07)} << uint32_t{(mask & 07)}; + + // Set some environment variables + putenv(const_cast(priority.c_str())); + putenv(const_cast(submitDir.c_str())); + putenv(const_cast(submitHost.c_str())); + putenv(const_cast(ss.str().c_str())); + } + + SlurmTaskExecutor::~SlurmTaskExecutor() + { + running_ = false; + monitorWorker_.join(); + } + + // Validates the job to ensure that all required values are set and are of the + // right type, + bool SlurmTaskExecutor::validateTaskParameters(const ConfigValues &job) + { + const std::unordered_set requiredFields{ + "minCPUs", "minMemoryMB", "minTmpDiskMB", + "priority", "timeLimitSeconds", "userID", + "workDir", "tmpDir", "command"}; + + for (const auto &requiredField : requiredFields) { + if (job.count(requiredField) == 0) { + throw std::runtime_error("Missing field " + requiredField); + } + } + return true; + } + + std::vector SlurmTaskExecutor::expandTaskParameters( + const ConfigValues &job, const ConfigValues &expansionValues) + { + std::vector newValues; + + const auto command = std::get(job.at("command")); + for (const auto &expandedCommand : + interpolateValues(command, expansionValues)) { + ConfigValues newCommand{job}; + newCommand.at("command") = expandedCommand; + newValues.emplace_back(newCommand); } - void readAndClean(const fs::path & fn, std::string & dest) { - if (! fs::exists(fn)) return; + return newValues; + } - std::ifstream ifh; - ifh.open(fn); - std::string contents(std::istreambuf_iterator{ifh}, {}); - ifh.close(); - fs::remove_all(fn); + std::future SlurmTaskExecutor::execute( + const std::string &taskName, const Task &task) + { + std::stringstream executorLog; - dest.swap(contents); + const auto &job = task.job; + const auto uniqueTaskName = taskName + "_" + getUniqueTag(6); + + fs::path tmpDir = std::get(job.at("tmpDir")); + std::string stdoutFile = (tmpDir / (uniqueTaskName + ".stdout")).string(); + std::string stderrFile = (tmpDir / (uniqueTaskName + ".stderr")).string(); + std::string workDir = std::get(job.at("workDir")); + + // Convert command to argc / argv + std::vector argv{nullptr}; + const auto command = + std::get>(task.job.at("command")); + std::transform( + command.begin(), command.end(), std::back_inserter(argv), + [](const std::string &s) { return const_cast(s.c_str()); }); + + char empty[] = ""; + char *env[1]; + env[0] = empty; + + char script[] = "#!/bin/bash\n$@\n"; + char stdinFile[] = "/dev/null"; + + // taken from slurm + int error_code; + job_desc_msg_t jd; + submit_response_msg_t *resp_msg; + + slurm_init_job_desc_msg(&jd); + jd.contiguous = 1; + jd.name = const_cast(taskName.c_str()); + jd.min_cpus = std::stoi(std::get(job.at("minCPUs"))); + + jd.pn_min_memory = std::stoi(std::get(job.at("minMemoryMB"))); + jd.pn_min_tmp_disk = + std::stoi(std::get(job.at("minTmpDiskMB"))); + jd.priority = std::stoi(std::get(job.at("priority"))); + jd.shared = 0; + jd.time_limit = + std::stoi(std::get(job.at("timeLimitSeconds"))); + jd.min_nodes = 1; + jd.user_id = std::stoi(std::get(job.at("userID"))); + jd.argv = argv.data(); + jd.argc = argv.size(); + // TODO figure out the script to run + jd.script = script; + jd.std_in = stdinFile; + jd.std_err = const_cast(stderrFile.c_str()); + jd.std_out = const_cast(stdoutFile.c_str()); + jd.work_dir = const_cast(workDir.c_str()); + jd.env_size = 1; + jd.environment = env; + + /* TODO: Add support for environment + jobDescription.env_size = 2; + env[0] = "SLURM_ENV_0=looking_good"; + env[1] = "SLURM_ENV_1=still_good"; + jobDescription.environment = env; + */ + + error_code = slurm_submit_batch_job(&jd, &resp_msg); + if (error_code) { + std::stringstream ss; + ss << "Unable to submit slurm job: " << slurm_strerror(error_code); + throw std::runtime_error(ss.str()); } - SlurmTaskExecutor::SlurmTaskExecutor() - : running_(true) - , monitorWorker_(&SlurmTaskExecutor::monitor, this) - { - std::string priority = "SLURM_PRIO_PROCESS=" + std::to_string(getpriority(PRIO_PROCESS, 0)); - std::string submitDir = "SLURM_SUBMIT_DIR=" + fs::current_path().string(); + uint32_t jobID = resp_msg->job_id; + executorLog << "Job " << resp_msg->job_submit_user_msg << '\n'; + slurm_free_submit_response_response_msg(resp_msg); - const size_t MAX_HOSTNAME_LENGTH = 50; - std::string submitHost(MAX_HOSTNAME_LENGTH, '\0'); - gethostname(submitHost.data(), MAX_HOSTNAME_LENGTH); - submitHost = "SLURM_SUBMIT_HOST=" + submitHost; - submitHost.resize(submitHost.find('\0')); + std::lock_guard lock(promiseGuard_); + Job newJob{.prom{}, .stdoutFile = stdoutFile, .stderrFile = stderrFile}; + auto fut = newJob.prom.get_future(); + runningJobs_.emplace(jobID, std::move(newJob)); - uint32_t mask = umask(0); - umask(mask); // Restore the old mask - - std::stringstream ss; - ss << "SLURM_UMASK=0" - << uint32_t{((mask >> 6) & 07)} - << uint32_t{((mask >> 3) & 07)} - << uint32_t{(mask & 07)}; - - // Set some environment variables - putenv(const_cast(priority.c_str())); - putenv(const_cast(submitDir.c_str())); - putenv(const_cast(submitHost.c_str())); - putenv(const_cast(ss.str().c_str())); - } - - SlurmTaskExecutor::~SlurmTaskExecutor() { - running_ = false; - monitorWorker_.join(); - } - - // Validates the job to ensure that all required values are set and are of the right type, - bool SlurmTaskExecutor::validateTaskParameters(const ConfigValues &job) { - const std::unordered_set requiredFields{ - "minCPUs", - "minMemoryMB", - "minTmpDiskMB", - "priority", - "timeLimitSeconds", - "userID", - "workDir", - "tmpDir", - "command" - }; - - for (const auto &requiredField: requiredFields) { - if (job.count(requiredField) == 0) { - throw std::runtime_error("Missing field " + requiredField); - } - } - return true; - } - - std::vector - SlurmTaskExecutor::expandTaskParameters(const ConfigValues &job, const ConfigValues &expansionValues) { - std::vector newValues; - - const auto command = std::get(job.at("command")); - for (const auto &expandedCommand: interpolateValues(command, expansionValues)) { - ConfigValues newCommand{job}; - newCommand.at("command") = expandedCommand; - newValues.emplace_back(newCommand); - } - - return newValues; - } - - std::future - SlurmTaskExecutor::execute(const std::string &taskName, const Task &task) { - std::stringstream executorLog; - - const auto &job = task.job; - const auto uniqueTaskName = taskName + "_" + getUniqueTag(6); - - fs::path tmpDir = std::get(job.at("tmpDir")); - std::string stdoutFile = (tmpDir / (uniqueTaskName + ".stdout")).string(); - std::string stderrFile = (tmpDir / (uniqueTaskName + ".stderr")).string(); - std::string workDir = std::get(job.at("workDir")); - - // Convert command to argc / argv - std::vector argv{nullptr}; - const auto command = std::get>(task.job.at("command")); - std::transform(command.begin(), - command.end(), - std::back_inserter(argv), - [](const std::string & s) { - return const_cast(s.c_str()); - }); - - char empty[] = ""; - char *env[1]; - env[0] = empty; - - char script[] = "#!/bin/bash\n$@\n"; - char stdinFile[] = "/dev/null"; - - // taken from slurm - int error_code; - job_desc_msg_t jd; - submit_response_msg_t *resp_msg; - - slurm_init_job_desc_msg(&jd); - jd.contiguous = 1; - jd.name = const_cast(taskName.c_str()); - jd.min_cpus = std::stoi(std::get(job.at("minCPUs"))); - - jd.pn_min_memory = std::stoi(std::get(job.at("minMemoryMB"))); - jd.pn_min_tmp_disk = std::stoi(std::get(job.at("minTmpDiskMB"))); - jd.priority = std::stoi(std::get(job.at("priority"))); - jd.shared = 0; - jd.time_limit = std::stoi(std::get(job.at("timeLimitSeconds"))); - jd.min_nodes = 1; - jd.user_id = std::stoi(std::get(job.at("userID"))); - jd.argv = argv.data(); - jd.argc = argv.size(); - // TODO figure out the script to run - jd.script = script; - jd.std_in = stdinFile; - jd.std_err = const_cast(stderrFile.c_str()); - jd.std_out = const_cast(stdoutFile.c_str()); - jd.work_dir = const_cast(workDir.c_str()); - jd.env_size = 1; - jd.environment = env; - - /* TODO: Add support for environment - jobDescription.env_size = 2; - env[0] = "SLURM_ENV_0=looking_good"; - env[1] = "SLURM_ENV_1=still_good"; - jobDescription.environment = env; - */ - - error_code = slurm_submit_batch_job(&jd, &resp_msg); - if (error_code) { - std::stringstream ss; - ss << "Unable to submit slurm job: " - << slurm_strerror(error_code); - throw std::runtime_error(ss.str()); - } - - uint32_t jobID = resp_msg->job_id; - executorLog << "Job " << resp_msg->job_submit_user_msg << '\n'; - slurm_free_submit_response_response_msg(resp_msg); + return fut; + } + void SlurmTaskExecutor::monitor() + { + std::unordered_set resolvedJobs; + while (running_) { + { std::lock_guard lock(promiseGuard_); - Job newJob{ - .prom{}, - .stdoutFile = stdoutFile, - .stderrFile = stderrFile - }; - auto fut = newJob.prom.get_future(); - runningJobs_.emplace(jobID, std::move(newJob)); + for (auto &[jobID, job] : runningJobs_) { + job_info_msg_t *jobStatus; + int error_code = + slurm_load_job(&jobStatus, jobID, SHOW_ALL | SHOW_DETAIL); + if (error_code != SLURM_SUCCESS) + continue; - return fut; - } + uint32_t idx = jobStatus->record_count; + if (idx == 0) + continue; + idx--; + const slurm_job_info_t &jobInfo = jobStatus->job_array[idx]; + AttemptRecord record; + switch (jobInfo.job_state) { + case JOB_PENDING: + case JOB_SUSPENDED: + case JOB_RUNNING: + continue; + break; + // Job has finished + case JOB_COMPLETE: /* completed execution successfully */ + case JOB_FAILED: /* completed execution unsuccessfully */ + record.executorLog = "Script errored.\n"; + break; + case JOB_CANCELLED: /* cancelled by user */ + record.executorLog = "Job cancelled by user.\n"; + break; + case JOB_TIMEOUT: /* terminated on reaching time limit */ + record.executorLog = "Job exceeded time limit.\n"; + break; + case JOB_NODE_FAIL: /* terminated on node failure */ + record.executorLog = "Node failed during execution\n"; + break; + case JOB_PREEMPTED: /* terminated due to preemption */ + record.executorLog = "Job terminated due to pre-emption.\n"; + break; + case JOB_BOOT_FAIL: /* terminated due to node boot failure */ + record.executorLog = + "Job failed to run due to failure of compute node to boot.\n"; + break; + case JOB_DEADLINE: /* terminated on deadline */ + record.executorLog = "Job terminated due to deadline.\n"; + break; + case JOB_OOM: /* experienced out of memory error */ + record.executorLog = "Job terminated due to out-of-memory.\n"; + break; + } + record.rc = jobInfo.exit_code; + slurm_free_job_info_msg(jobStatus); - void SlurmTaskExecutor::monitor() { - std::unordered_set resolvedJobs; - while (running_) { - { - std::lock_guard lock(promiseGuard_); - for (auto & [jobID, job] : runningJobs_) { - job_info_msg_t * jobStatus; - int error_code = slurm_load_job(&jobStatus, jobID, SHOW_ALL | SHOW_DETAIL); - if (error_code != SLURM_SUCCESS) continue; + readAndClean(job.stdoutFile, record.outputLog); + readAndClean(job.stderrFile, record.errorLog); - uint32_t idx = jobStatus->record_count; - if (idx == 0) continue; - idx--; - const slurm_job_info_t & jobInfo = jobStatus->job_array[idx]; - AttemptRecord record; - switch(jobInfo.job_state) { - case JOB_PENDING: - case JOB_SUSPENDED: - case JOB_RUNNING: - continue; - break; - // Job has finished - case JOB_COMPLETE: /* completed execution successfully */ - case JOB_FAILED: /* completed execution unsuccessfully */ - record.executorLog = "Script errored.\n"; - break; - case JOB_CANCELLED: /* cancelled by user */ - record.executorLog = "Job cancelled by user.\n"; - break; - case JOB_TIMEOUT: /* terminated on reaching time limit */ - record.executorLog = "Job exceeded time limit.\n"; - break; - case JOB_NODE_FAIL: /* terminated on node failure */ - record.executorLog = "Node failed during execution\n"; - break; - case JOB_PREEMPTED: /* terminated due to preemption */ - record.executorLog = "Job terminated due to pre-emption.\n"; - break; - case JOB_BOOT_FAIL: /* terminated due to node boot failure */ - record.executorLog = "Job failed to run due to failure of compute node to boot.\n"; - break; - case JOB_DEADLINE: /* terminated on deadline */ - record.executorLog = "Job terminated due to deadline.\n"; - break; - case JOB_OOM: /* experienced out of memory error */ - record.executorLog = "Job terminated due to out-of-memory.\n"; - break; - } - record.rc = jobInfo.exit_code; - slurm_free_job_info_msg(jobStatus); - - readAndClean(job.stdoutFile, record.outputLog); - readAndClean(job.stderrFile, record.errorLog); - - job.prom.set_value(std::move(record)); - resolvedJobs.insert(jobID); - } - - for (const auto &jobID : resolvedJobs) { - runningJobs_.extract(jobID); - } - } - - std::this_thread::sleep_for(std::chrono::milliseconds(250)); + job.prom.set_value(std::move(record)); + resolvedJobs.insert(jobID); } + + for (const auto &jobID : resolvedJobs) { + runningJobs_.extract(jobID); + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(250)); } -} + } +} // namespace daggy::executors::task #endif diff --git a/daggy/src/loggers/dag_run/FileSystemLogger.cpp b/daggy/src/loggers/dag_run/FileSystemLogger.cpp index 253cabf..3e8eb22 100644 --- a/daggy/src/loggers/dag_run/FileSystemLogger.cpp +++ b/daggy/src/loggers/dag_run/FileSystemLogger.cpp @@ -1,178 +1,212 @@ -#include -#include - #include -#include #include #include +#include +#include +#include namespace fs = std::filesystem; using namespace daggy::loggers::dag_run; namespace daggy { - inline const fs::path FileSystemLogger::getCurrentPath() const { return root_ / "current"; } + inline const fs::path FileSystemLogger::getCurrentPath() const + { + return root_ / "current"; + } - inline const fs::path FileSystemLogger::getRunsRoot() const { return root_ / "runs"; } + inline const fs::path FileSystemLogger::getRunsRoot() const + { + return root_ / "runs"; + } - inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const { - return getRunsRoot() / std::to_string(runID); + inline const fs::path FileSystemLogger::getRunRoot(DAGRunID runID) const + { + return getRunsRoot() / std::to_string(runID); + } + + FileSystemLogger::FileSystemLogger(fs::path root) + : root_(root) + , nextRunID_(0) + { + const std::vector reqPaths{root_, getCurrentPath(), + getRunsRoot()}; + for (const auto &path : reqPaths) { + if (!fs::exists(path)) { + fs::create_directories(path); + } } - FileSystemLogger::FileSystemLogger(fs::path root) - : root_(root), nextRunID_(0) { - const std::vector reqPaths{root_, getCurrentPath(), getRunsRoot()}; - for (const auto &path: reqPaths) { - if (!fs::exists(path)) { fs::create_directories(path); } - } + // Get the next run ID + for (auto &dir : fs::directory_iterator(getRunsRoot())) { + try { + size_t runID = std::stoull(dir.path().stem()); + if (runID > nextRunID_) + nextRunID_ = runID + 1; + } + catch (std::exception &e) { + continue; + } + } + } - // Get the next run ID - for (auto &dir: fs::directory_iterator(getRunsRoot())) { - try { - size_t runID = std::stoull(dir.path().stem()); - if (runID > nextRunID_) nextRunID_ = runID + 1; - } catch (std::exception &e) { - continue; - } - } + // Execution + DAGRunID FileSystemLogger::startDAGRun(std::string name, const TaskSet &tasks) + { + DAGRunID runID = nextRunID_++; + + // TODO make this threadsafe + fs::path runDir = getRunRoot(runID); + // std::lock_guard guard(runLocks[runDir]); + + // Init the directory + fs::path runRoot = getRunsRoot() / std::to_string(runID); + fs::create_directories(runRoot); + + // Create meta.json with DAGRun Name and task definitions + std::ofstream ofh(runRoot / "metadata.json", + std::ios::trunc | std::ios::binary); + ofh << R"({ "name": )" << std::quoted(name) << R"(, "tasks": )" + << tasksToJSON(tasks) << "}\n"; + ofh.close(); + + // Task directories + for (const auto &[name, task] : tasks) { + auto taskDir = runRoot / name; + fs::create_directories(taskDir); + std::ofstream ofh(taskDir / "states.csv"); } - // Execution - DAGRunID FileSystemLogger::startDAGRun(std::string name, const TaskSet &tasks) { - DAGRunID runID = nextRunID_++; + return runID; + } - // TODO make this threadsafe - fs::path runDir = getRunRoot(runID); - // std::lock_guard guard(runLocks[runDir]); + void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) + { + std::ofstream ofh(getRunRoot(dagRunID) / "states.csv", + std::ios::binary | std::ios::app); + ofh << std::quoted(timePointToString(Clock::now())) << ',' + << state._to_string() << '\n'; + ofh.flush(); + ofh.close(); + } - // Init the directory - fs::path runRoot = getRunsRoot() / std::to_string(runID); - fs::create_directories(runRoot); - - // Create meta.json with DAGRun Name and task definitions - std::ofstream ofh(runRoot / "metadata.json", std::ios::trunc | std::ios::binary); - ofh << R"({ "name": )" << std::quoted(name) << R"(, "tasks": )" << tasksToJSON(tasks) << "}\n"; - ofh.close(); - - // Task directories - for (const auto &[name, task]: tasks) { - auto taskDir = runRoot / name; - fs::create_directories(taskDir); - std::ofstream ofh(taskDir / "states.csv"); - } - - return runID; + void FileSystemLogger::logTaskAttempt(DAGRunID dagRunID, + const std::string &taskName, + const AttemptRecord &attempt) + { + auto taskRoot = getRunRoot(dagRunID) / taskName; + size_t i = 1; + while (fs::exists(taskRoot / std::to_string(i))) { + ++i; } - void FileSystemLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) { - std::ofstream ofh(getRunRoot(dagRunID) / "states.csv", std::ios::binary | std::ios::app); - ofh << std::quoted(timePointToString(Clock::now())) << ',' << state._to_string() << '\n'; - ofh.flush(); - ofh.close(); + auto attemptDir = taskRoot / std::to_string(i); + fs::create_directories(attemptDir); + + std::ofstream ofh; + + // Metadata + ofh.open(attemptDir / "metadata.json"); + ofh << "{\n" + << R"("startTime": )" + << std::quoted(timePointToString(attempt.startTime)) << ",\n" + << R"("stopTime": )" << std::quoted(timePointToString(attempt.stopTime)) + << ",\n" + << R"("rc": )" << attempt.rc << '\n' + << '}'; + + // output + ofh.open(attemptDir / "executor.log"); + ofh << attempt.executorLog << std::flush; + ofh.close(); + + // Output + ofh.open(attemptDir / "output.log"); + ofh << attempt.outputLog << std::flush; + ofh.close(); + + // Error + ofh.open(attemptDir / "error.log"); + ofh << attempt.errorLog << std::flush; + ofh.close(); + } + + void FileSystemLogger::updateTaskState(DAGRunID dagRunID, + const std::string &taskName, + RunState state) + { + std::ofstream ofh(getRunRoot(dagRunID) / taskName / "states.csv", + std::ios::binary | std::ios::app); + ofh << std::quoted(timePointToString(Clock::now())) << ',' + << state._to_string() << '\n'; + ofh.flush(); + ofh.close(); + } + + // Querying + std::vector FileSystemLogger::getDAGs(uint32_t stateMask) + { + return {}; + } + + DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunID) + { + DAGRunRecord record; + auto runRoot = getRunRoot(dagRunID); + if (!fs::exists(runRoot)) { + throw std::runtime_error("No DAGRun with that ID exists"); } - void - FileSystemLogger::logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, - const AttemptRecord &attempt) { - auto taskRoot = getRunRoot(dagRunID) / taskName; - size_t i = 1; - while (fs::exists(taskRoot / std::to_string(i))) { ++i; } + std::ifstream ifh(runRoot / "metadata.json", std::ios::binary); + std::string metaData; + std::getline(ifh, metaData, '\0'); + ifh.close(); - auto attemptDir = taskRoot / std::to_string(i); - fs::create_directories(attemptDir); + rj::Document doc; + doc.Parse(metaData.c_str()); - std::ofstream ofh; + record.name = doc["name"].GetString(); + record.tasks = tasksFromJSON(doc["tasks"]); - // Metadata - ofh.open(attemptDir / "metadata.json"); - ofh << "{\n" - << R"("startTime": )" << std::quoted(timePointToString(attempt.startTime)) << ",\n" - << R"("stopTime": )" << std::quoted(timePointToString(attempt.stopTime)) << ",\n" - << R"("rc": )" << attempt.rc << '\n' - << '}'; + // DAG State Changes + std::string line; + std::string token; + auto dagStateFile = runRoot / "states.csv"; + ifh.open(dagStateFile); + while (std::getline(ifh, line)) { + std::stringstream ss{line}; + std::string time; + std::string state; + std::getline(ss, time, ','); + std::getline(ss, state); - // output - ofh.open(attemptDir / "executor.log"); - ofh << attempt.executorLog << std::flush; - ofh.close(); - - // Output - ofh.open(attemptDir / "output.log"); - ofh << attempt.outputLog << std::flush; - ofh.close(); - - // Error - ofh.open(attemptDir / "error.log"); - ofh << attempt.errorLog << std::flush; - ofh.close(); + record.dagStateChanges.emplace_back( + DAGUpdateRecord{.time = stringToTimePoint(time), + .newState = RunState::_from_string(state.c_str())}); } + ifh.close(); - void FileSystemLogger::updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) { - std::ofstream ofh(getRunRoot(dagRunID) / taskName / "states.csv", std::ios::binary | std::ios::app); - ofh << std::quoted(timePointToString(Clock::now())) << ',' << state._to_string() << '\n'; - ofh.flush(); - ofh.close(); + // Task states + for (const auto &[taskName, task] : record.tasks) { + auto taskStateFile = runRoot / taskName / "states.csv"; + if (!fs::exists(taskStateFile)) { + record.taskRunStates.emplace(taskName, RunState::QUEUED); + continue; + } + + ifh.open(taskStateFile); + while (std::getline(ifh, line)) { + continue; + } + std::stringstream ss{line}; + while (std::getline(ss, token, ',')) { + continue; + } + RunState taskState = RunState::_from_string(token.c_str()); + record.taskRunStates.emplace(taskName, taskState); + ifh.close(); } - - // Querying - std::vector FileSystemLogger::getDAGs(uint32_t stateMask) { - return {}; - } - - DAGRunRecord FileSystemLogger::getDAGRun(DAGRunID dagRunID) { - DAGRunRecord record; - auto runRoot = getRunRoot(dagRunID); - if (!fs::exists(runRoot)) { - throw std::runtime_error("No DAGRun with that ID exists"); - } - - std::ifstream ifh(runRoot / "metadata.json", std::ios::binary); - std::string metaData; - std::getline(ifh, metaData, '\0'); - ifh.close(); - - rj::Document doc; - doc.Parse(metaData.c_str()); - - record.name = doc["name"].GetString(); - record.tasks = tasksFromJSON(doc["tasks"]); - - // DAG State Changes - std::string line; - std::string token; - auto dagStateFile = runRoot / "states.csv"; - ifh.open(dagStateFile); - while (std::getline(ifh, line)) { - std::stringstream ss{line}; - std::string time; - std::string state; - std::getline(ss, time, ','); - std::getline(ss, state); - - record.dagStateChanges.emplace_back(DAGUpdateRecord{ - .time = stringToTimePoint(time), - .newState = RunState::_from_string(state.c_str()) - }); - } - ifh.close(); - - // Task states - for (const auto &[taskName, task]: record.tasks) { - auto taskStateFile = runRoot / taskName / "states.csv"; - if (!fs::exists(taskStateFile)) { - record.taskRunStates.emplace(taskName, RunState::QUEUED); - continue; - } - - ifh.open(taskStateFile); - while (std::getline(ifh, line)) { continue; } - std::stringstream ss{line}; - while (std::getline(ss, token, ',')) { continue; } - RunState taskState = RunState::_from_string(token.c_str()); - record.taskRunStates.emplace(taskName, taskState); - ifh.close(); - } - return record; - } -} + return record; + } +} // namespace daggy diff --git a/daggy/src/loggers/dag_run/OStreamLogger.cpp b/daggy/src/loggers/dag_run/OStreamLogger.cpp index fa70239..a56d29c 100644 --- a/daggy/src/loggers/dag_run/OStreamLogger.cpp +++ b/daggy/src/loggers/dag_run/OStreamLogger.cpp @@ -1,122 +1,135 @@ -#include -#include - #include -#include +#include #include +#include +#include -namespace daggy { - namespace loggers { - namespace dag_run { - OStreamLogger::OStreamLogger(std::ostream &os) : os_(os) {} +namespace daggy { namespace loggers { namespace dag_run { + OStreamLogger::OStreamLogger(std::ostream &os) + : os_(os) + { + } - // Execution - DAGRunID OStreamLogger::startDAGRun(std::string name, const TaskSet &tasks) { - std::lock_guard lock(guard_); - size_t runID = dagRuns_.size(); - dagRuns_.push_back({ - .name = name, - .tasks = tasks - }); - for (const auto &[name, _]: tasks) { - _updateTaskState(runID, name, RunState::QUEUED); - } - _updateDAGRunState(runID, RunState::QUEUED); - - os_ << "Starting new DAGRun named " << name << " with ID " << runID << " and " << tasks.size() - << " tasks" << std::endl; - for (const auto &[name, task]: tasks) { - os_ << "TASK (" << name << "): " << configToJSON(task.job); - os_ << std::endl; - } - return runID; - } - - void OStreamLogger::addTask(DAGRunID dagRunID, const std::string taskName, const Task &task) { - std::lock_guard lock(guard_); - auto &dagRun = dagRuns_[dagRunID]; - dagRun.tasks[taskName] = task; - _updateTaskState(dagRunID, taskName, RunState::QUEUED); - } - - void OStreamLogger::updateTask(DAGRunID dagRunID, const std::string taskName, const Task &task) { - std::lock_guard lock(guard_); - auto &dagRun = dagRuns_[dagRunID]; - dagRun.tasks[taskName] = task; - } - - void OStreamLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) { - std::lock_guard lock(guard_); - _updateDAGRunState(dagRunID, state); - } - - void OStreamLogger::_updateDAGRunState(DAGRunID dagRunID, RunState state) { - os_ << "DAG State Change(" << dagRunID << "): " << state._to_string() << std::endl; - dagRuns_[dagRunID].dagStateChanges.push_back({Clock::now(), state}); - } - - void OStreamLogger::logTaskAttempt(DAGRunID dagRunID, const std::string &taskName, - const AttemptRecord &attempt) { - std::lock_guard lock(guard_); - const std::string &msg = attempt.rc == 0 ? attempt.outputLog : attempt.errorLog; - os_ << "Task Attempt (" << dagRunID << '/' << taskName << "): Ran with RC " << attempt.rc << ": " - << msg << std::endl; - - dagRuns_[dagRunID].taskAttempts[taskName].push_back(attempt); - } - - void OStreamLogger::updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) { - std::lock_guard lock(guard_); - _updateTaskState(dagRunID, taskName, state); - } - - void OStreamLogger::_updateTaskState(DAGRunID dagRunID, const std::string &taskName, RunState state) { - auto &dagRun = dagRuns_.at(dagRunID); - dagRun.taskStateChanges.push_back({Clock::now(), taskName, state}); - auto it = dagRun.taskRunStates.find(taskName); - if (it == dagRun.taskRunStates.end()) { - dagRun.taskRunStates.emplace(taskName, state); - } else { - it->second = state; - } - - os_ << "Task State Change (" << dagRunID << '/' << taskName << "): " - << state._to_string() - << std::endl; - } - - // Querying - std::vector OStreamLogger::getDAGs(uint32_t stateMask) { - std::vector summaries; - std::lock_guard lock(guard_); - size_t i = 0; - for (const auto &run: dagRuns_) { - DAGRunSummary summary{ - .runID = i, - .name = run.name, - .runState = run.dagStateChanges.back().newState, - .startTime = run.dagStateChanges.front().time, - .lastUpdate = std::max(run.taskStateChanges.back().time, - run.dagStateChanges.back().time) - }; - - for (const auto &[_, taskState]: run.taskRunStates) { - summary.taskStateCounts[taskState]++; - } - - summaries.emplace_back(summary); - } - return summaries; - } - - DAGRunRecord OStreamLogger::getDAGRun(DAGRunID dagRunID) { - if (dagRunID >= dagRuns_.size()) { - throw std::runtime_error("No such DAGRun ID"); - } - std::lock_guard lock(guard_); - return dagRuns_[dagRunID]; - } - } + // Execution + DAGRunID OStreamLogger::startDAGRun(std::string name, const TaskSet &tasks) + { + std::lock_guard lock(guard_); + size_t runID = dagRuns_.size(); + dagRuns_.push_back({.name = name, .tasks = tasks}); + for (const auto &[name, _] : tasks) { + _updateTaskState(runID, name, RunState::QUEUED); } -} + _updateDAGRunState(runID, RunState::QUEUED); + + os_ << "Starting new DAGRun named " << name << " with ID " << runID + << " and " << tasks.size() << " tasks" << std::endl; + for (const auto &[name, task] : tasks) { + os_ << "TASK (" << name << "): " << configToJSON(task.job); + os_ << std::endl; + } + return runID; + } + + void OStreamLogger::addTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) + { + std::lock_guard lock(guard_); + auto &dagRun = dagRuns_[dagRunID]; + dagRun.tasks[taskName] = task; + _updateTaskState(dagRunID, taskName, RunState::QUEUED); + } + + void OStreamLogger::updateTask(DAGRunID dagRunID, const std::string taskName, + const Task &task) + { + std::lock_guard lock(guard_); + auto &dagRun = dagRuns_[dagRunID]; + dagRun.tasks[taskName] = task; + } + + void OStreamLogger::updateDAGRunState(DAGRunID dagRunID, RunState state) + { + std::lock_guard lock(guard_); + _updateDAGRunState(dagRunID, state); + } + + void OStreamLogger::_updateDAGRunState(DAGRunID dagRunID, RunState state) + { + os_ << "DAG State Change(" << dagRunID << "): " << state._to_string() + << std::endl; + dagRuns_[dagRunID].dagStateChanges.push_back({Clock::now(), state}); + } + + void OStreamLogger::logTaskAttempt(DAGRunID dagRunID, + const std::string &taskName, + const AttemptRecord &attempt) + { + std::lock_guard lock(guard_); + const std::string &msg = + attempt.rc == 0 ? attempt.outputLog : attempt.errorLog; + os_ << "Task Attempt (" << dagRunID << '/' << taskName << "): Ran with RC " + << attempt.rc << ": " << msg << std::endl; + + dagRuns_[dagRunID].taskAttempts[taskName].push_back(attempt); + } + + void OStreamLogger::updateTaskState(DAGRunID dagRunID, + const std::string &taskName, + RunState state) + { + std::lock_guard lock(guard_); + _updateTaskState(dagRunID, taskName, state); + } + + void OStreamLogger::_updateTaskState(DAGRunID dagRunID, + const std::string &taskName, + RunState state) + { + auto &dagRun = dagRuns_.at(dagRunID); + dagRun.taskStateChanges.push_back({Clock::now(), taskName, state}); + auto it = dagRun.taskRunStates.find(taskName); + if (it == dagRun.taskRunStates.end()) { + dagRun.taskRunStates.emplace(taskName, state); + } + else { + it->second = state; + } + + os_ << "Task State Change (" << dagRunID << '/' << taskName + << "): " << state._to_string() << std::endl; + } + + // Querying + std::vector OStreamLogger::getDAGs(uint32_t stateMask) + { + std::vector summaries; + std::lock_guard lock(guard_); + size_t i = 0; + for (const auto &run : dagRuns_) { + DAGRunSummary summary{ + .runID = i, + .name = run.name, + .runState = run.dagStateChanges.back().newState, + .startTime = run.dagStateChanges.front().time, + .lastUpdate = std::max(run.taskStateChanges.back().time, + run.dagStateChanges.back().time)}; + + for (const auto &[_, taskState] : run.taskRunStates) { + summary.taskStateCounts[taskState]++; + } + + summaries.emplace_back(summary); + } + return summaries; + } + + DAGRunRecord OStreamLogger::getDAGRun(DAGRunID dagRunID) + { + if (dagRunID >= dagRuns_.size()) { + throw std::runtime_error("No such DAGRun ID"); + } + std::lock_guard lock(guard_); + return dagRuns_[dagRunID]; + } +}}} // namespace daggy::loggers::dag_run diff --git a/tests/int_basic.cpp b/tests/int_basic.cpp index bf3d138..5088e5a 100644 --- a/tests/int_basic.cpp +++ b/tests/int_basic.cpp @@ -1,9 +1,9 @@ +#include #include #include "daggy/DAG.hpp" -#include - -TEST_CASE("General tests", "[general]") { - REQUIRE(1 == 1); +TEST_CASE("General tests", "[general]") +{ + REQUIRE(1 == 1); } diff --git a/tests/main.cpp b/tests/main.cpp index ab87a90..4387fa6 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -6,8 +6,9 @@ #include -TEST_CASE("Sanity tests", "[sanity]") { - REQUIRE(1 == 1); +TEST_CASE("Sanity tests", "[sanity]") +{ + REQUIRE(1 == 1); } // compile and run diff --git a/tests/unit_dag.cpp b/tests/unit_dag.cpp index ed29bd9..b2e0392 100644 --- a/tests/unit_dag.cpp +++ b/tests/unit_dag.cpp @@ -1,90 +1,87 @@ +#include #include #include "daggy/DAG.hpp" -#include +TEST_CASE("dag_construction", "[dag]") +{ + daggy::DAG dag; -TEST_CASE("dag_construction", "[dag]") { - daggy::DAG dag; + REQUIRE(dag.size() == 0); + REQUIRE(dag.empty()); - REQUIRE(dag.size() == 0); - REQUIRE(dag.empty()); + REQUIRE_NOTHROW(dag.addVertex(0, 0)); + for (size_t i = 1; i < 10; ++i) { + dag.addVertex(i, i); + REQUIRE(dag.hasVertex(i)); + REQUIRE(dag.getVertex(i).data == i); + dag.addEdge(i - 1, i); + } - REQUIRE_NOTHROW(dag.addVertex(0, 0)); - for (size_t i = 1; i < 10; ++i) { - dag.addVertex(i, i); - REQUIRE(dag.hasVertex(i)); - REQUIRE(dag.getVertex(i).data == i); - dag.addEdge(i - 1, i); - } + REQUIRE(dag.size() == 10); + REQUIRE(!dag.empty()); - REQUIRE(dag.size() == 10); - REQUIRE(!dag.empty()); + // Cannot add an edge that would result in a cycle + dag.addEdge(9, 5); + REQUIRE(!dag.isValid()); - // Cannot add an edge that would result in a cycle - dag.addEdge(9, 5); - REQUIRE(!dag.isValid()); - - // Bounds checking - SECTION("addEdge Bounds Checking") { - REQUIRE_THROWS(dag.addEdge(20, 0)); - REQUIRE_THROWS(dag.addEdge(0, 20)); - } + // Bounds checking + SECTION("addEdge Bounds Checking") + { + REQUIRE_THROWS(dag.addEdge(20, 0)); + REQUIRE_THROWS(dag.addEdge(0, 20)); + } } -TEST_CASE("dag_traversal", "[dag]") { - daggy::DAG dag; +TEST_CASE("dag_traversal", "[dag]") +{ + daggy::DAG dag; - const int N_VERTICES = 10; + const int N_VERTICES = 10; - for (int i = 0; i < N_VERTICES; ++i) { dag.addVertex(i, i); } + for (int i = 0; i < N_VERTICES; ++i) { + dag.addVertex(i, i); + } - /* - 0 ---------------------\ - 1 ---------- \ \ /-----> 8 - 2 ---- 3 ---- > 5 -------> 6 -----> 7 - 4 -------------------------------/ \-----> 9 - */ + /* + 0 ---------------------\ + 1 ---------- \ \ /-----> 8 + 2 ---- 3 ---- > 5 -------> 6 -----> 7 + 4 -------------------------------/ \-----> 9 + */ - std::vector> edges{ - {0, 6}, - {1, 5}, - {5, 6}, - {6, 7}, - {2, 3}, - {3, 5}, - {4, 7}, - {7, 8}, - {7, 9} - }; + std::vector> edges{{0, 6}, {1, 5}, {5, 6}, {6, 7}, {2, 3}, + {3, 5}, {4, 7}, {7, 8}, {7, 9}}; - for (auto const[from, to]: edges) { - dag.addEdge(from, to); + for (auto const [from, to] : edges) { + dag.addEdge(from, to); + } + + SECTION("Basic Traversal") + { + dag.reset(); + std::vector visitOrder(N_VERTICES); + size_t i = 0; + while (!dag.allVisited()) { + const auto v = dag.visitNext().value(); + dag.completeVisit(v.first); + visitOrder[v.first] = i; + ++i; } - SECTION("Basic Traversal") { - dag.reset(); - std::vector visitOrder(N_VERTICES); - size_t i = 0; - while (!dag.allVisited()) { - const auto v = dag.visitNext().value(); - dag.completeVisit(v.first); - visitOrder[v.first] = i; - ++i; - } - - // Ensure visit order is preserved - for (auto const[from, to]: edges) { - REQUIRE(visitOrder[from] <= visitOrder[to]); - } + // Ensure visit order is preserved + for (auto const [from, to] : edges) { + REQUIRE(visitOrder[from] <= visitOrder[to]); } + } - SECTION("Iteration") { - size_t nVisited = 0; - dag.forEach([&](auto &k) { - (void) k; - ++nVisited; - }); - REQUIRE(nVisited == dag.size()); - } + SECTION("Iteration") + { + size_t nVisited = 0; + dag.forEach([&](auto &k) { + (void)k; + ++nVisited; + }); + REQUIRE(nVisited == dag.size()); + } } diff --git a/tests/unit_dagrun_loggers.cpp b/tests/unit_dagrun_loggers.cpp index 1b5cf29..dc9a72a 100644 --- a/tests/unit_dagrun_loggers.cpp +++ b/tests/unit_dagrun_loggers.cpp @@ -1,8 +1,7 @@ -#include +#include #include #include - -#include +#include #include "daggy/loggers/dag_run/FileSystemLogger.hpp" #include "daggy/loggers/dag_run/OStreamLogger.hpp" @@ -13,25 +12,32 @@ using namespace daggy; using namespace daggy::loggers::dag_run; const TaskSet SAMPLE_TASKS{ - {"work_a", Task{.job{{"command", std::vector{"/bin/echo", "a"}}}, .children{"c"}}}, - {"work_b", Task{.job{{"command", std::vector{"/bin/echo", "b"}}}, .children{"c"}}}, - {"work_c", Task{.job{{"command", std::vector{"/bin/echo", "c"}}}}} -}; + {"work_a", + Task{.job{{"command", std::vector{"/bin/echo", "a"}}}, + .children{"c"}}}, + {"work_b", + Task{.job{{"command", std::vector{"/bin/echo", "b"}}}, + .children{"c"}}}, + {"work_c", + Task{.job{{"command", std::vector{"/bin/echo", "c"}}}}}}; -inline DAGRunID testDAGRunInit(DAGRunLogger &logger, const std::string &name, const TaskSet &tasks) { - auto runID = logger.startDAGRun(name, tasks); - auto dagRun = logger.getDAGRun(runID); +inline DAGRunID testDAGRunInit(DAGRunLogger &logger, const std::string &name, + const TaskSet &tasks) +{ + auto runID = logger.startDAGRun(name, tasks); + auto dagRun = logger.getDAGRun(runID); - REQUIRE(dagRun.tasks == tasks); + REQUIRE(dagRun.tasks == tasks); - REQUIRE(dagRun.taskRunStates.size() == tasks.size()); - auto nonQueuedTask = std::find_if(dagRun.taskRunStates.begin(), dagRun.taskRunStates.end(), - [](const auto &a) { return a.second != +RunState::QUEUED; }); - REQUIRE(nonQueuedTask == dagRun.taskRunStates.end()); + REQUIRE(dagRun.taskRunStates.size() == tasks.size()); + auto nonQueuedTask = + std::find_if(dagRun.taskRunStates.begin(), dagRun.taskRunStates.end(), + [](const auto &a) { return a.second != +RunState::QUEUED; }); + REQUIRE(nonQueuedTask == dagRun.taskRunStates.end()); - REQUIRE(dagRun.dagStateChanges.size() == 1); - REQUIRE(dagRun.dagStateChanges.back().newState == +RunState::QUEUED); - return runID; + REQUIRE(dagRun.dagStateChanges.size() == 1); + REQUIRE(dagRun.dagStateChanges.back().newState == +RunState::QUEUED); + return runID; } /* @@ -54,14 +60,16 @@ TEST_CASE("Filesystem Logger", "[filesystem_logger]") { } */ -TEST_CASE("ostream_logger", "[ostream_logger]") { - //cleanup(); - std::stringstream ss; - daggy::loggers::dag_run::OStreamLogger logger(ss); +TEST_CASE("ostream_logger", "[ostream_logger]") +{ + // cleanup(); + std::stringstream ss; + daggy::loggers::dag_run::OStreamLogger logger(ss); - SECTION("DAGRun Starts") { - testDAGRunInit(logger, "init_test", SAMPLE_TASKS); - } + SECTION("DAGRun Starts") + { + testDAGRunInit(logger, "init_test", SAMPLE_TASKS); + } - // cleanup(); + // cleanup(); } diff --git a/tests/unit_executor_forkingexecutor.cpp b/tests/unit_executor_forkingexecutor.cpp index 5e6195d..42e393d 100644 --- a/tests/unit_executor_forkingexecutor.cpp +++ b/tests/unit_executor_forkingexecutor.cpp @@ -1,86 +1,103 @@ -#include +#include #include +#include -#include "daggy/executors/task/ForkingTaskExecutor.hpp" #include "daggy/Serialization.hpp" #include "daggy/Utilities.hpp" +#include "daggy/executors/task/ForkingTaskExecutor.hpp" -#include +TEST_CASE("forking_executor", "[forking_executor]") +{ + daggy::executors::task::ForkingTaskExecutor ex(10); -TEST_CASE("forking_executor", "[forking_executor]") { - daggy::executors::task::ForkingTaskExecutor ex(10); + SECTION("Simple Run") + { + daggy::Task task{ + .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ + "/usr/bin/echo", "abc", "123"}}}}; - SECTION("Simple Run") { - daggy::Task task{.job{ - {"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/echo", "abc", "123"}}}}; + REQUIRE(ex.validateTaskParameters(task.job)); - REQUIRE(ex.validateTaskParameters(task.job)); + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); + REQUIRE(rec.rc == 0); + REQUIRE(rec.outputLog.size() >= 6); + REQUIRE(rec.errorLog.empty()); + } - REQUIRE(rec.rc == 0); - REQUIRE(rec.outputLog.size() >= 6); - REQUIRE(rec.errorLog.empty()); + SECTION("Error Run") + { + daggy::Task task{ + .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ + "/usr/bin/expr", "1", "+", "+"}}}}; + + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); + + REQUIRE(rec.rc == 2); + REQUIRE(rec.errorLog.size() >= 20); + REQUIRE(rec.outputLog.empty()); + } + + SECTION("Large Output") + { + const std::vector BIG_FILES{"/usr/share/dict/linux.words", + "/usr/share/dict/cracklib-small", + "/etc/ssh/moduli"}; + + for (const auto &bigFile : BIG_FILES) { + if (!std::filesystem::exists(bigFile)) + continue; + + daggy::Task task{ + .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ + "/usr/bin/cat", bigFile}}}}; + + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); + + REQUIRE(rec.rc == 0); + REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile)); + REQUIRE(rec.errorLog.empty()); } + } - SECTION("Error Run") { - daggy::Task task{.job{ - {"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}}; + SECTION("Parameter Expansion") + { + std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; + auto params = daggy::configFromJSON(testParams); - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); + std::string taskJSON = + R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; + auto tasks = daggy::tasksFromJSON(taskJSON); - REQUIRE(rec.rc == 2); - REQUIRE(rec.errorLog.size() >= 20); - REQUIRE(rec.outputLog.empty()); - } + auto result = daggy::expandTaskSet(tasks, ex, params); + REQUIRE(result.size() == 2); + } - SECTION("Large Output") { - const std::vector BIG_FILES{ - "/usr/share/dict/linux.words", "/usr/share/dict/cracklib-small", "/etc/ssh/moduli" - }; + SECTION("Build with expansion") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; + auto params = daggy::configFromJSON(testParams); + std::string testTasks = + R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; + auto tasks = + daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params); + REQUIRE(tasks.size() == 4); + } - for (const auto &bigFile: BIG_FILES) { - if (!std::filesystem::exists(bigFile)) continue; + SECTION("Build with expansion using parents instead of children") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; + auto params = daggy::configFromJSON(testParams); + std::string testTasks = + R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})"; + auto tasks = + daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params); - daggy::Task task{.job{ - {"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/cat", bigFile}}}}; - - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); - - REQUIRE(rec.rc == 0); - REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile)); - REQUIRE(rec.errorLog.empty()); - } - } - - SECTION("Parameter Expansion") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; - auto params = daggy::configFromJSON(testParams); - - std::string taskJSON = R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; - auto tasks = daggy::tasksFromJSON(taskJSON); - - auto result = daggy::expandTaskSet(tasks, ex, params); - REQUIRE(result.size() == 2); - } - - SECTION("Build with expansion") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; - auto params = daggy::configFromJSON(testParams); - std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; - auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params); - REQUIRE(tasks.size() == 4); - } - - SECTION("Build with expansion using parents instead of children") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; - auto params = daggy::configFromJSON(testParams); - std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})"; - auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks), ex, params); - - REQUIRE(tasks.size() == 4); - } + REQUIRE(tasks.size() == 4); + } } diff --git a/tests/unit_executor_slurmexecutor.cpp b/tests/unit_executor_slurmexecutor.cpp index 20151e8..3695f89 100644 --- a/tests/unit_executor_slurmexecutor.cpp +++ b/tests/unit_executor_slurmexecutor.cpp @@ -1,111 +1,124 @@ -#include -#include - -#include #include - -#include "daggy/executors/task/SlurmTaskExecutor.hpp" -#include "daggy/Serialization.hpp" -#include "daggy/Utilities.hpp" +#include #include +#include +#include + +#include "daggy/Serialization.hpp" +#include "daggy/Utilities.hpp" +#include "daggy/executors/task/SlurmTaskExecutor.hpp" namespace fs = std::filesystem; #ifdef DAGGY_ENABLE_SLURM -TEST_CASE("slurm_execution", "[slurm_executor]") { - daggy::executors::task::SlurmTaskExecutor ex; +TEST_CASE("slurm_execution", "[slurm_executor]") +{ + daggy::executors::task::SlurmTaskExecutor ex; - daggy::ConfigValues defaultJobValues{ - {"minCPUs", "1"}, - {"minMemoryMB", "100"}, - {"minTmpDiskMB", "0"}, - {"priority", "1"}, - {"timeLimitSeconds", "200"}, - {"userID", std::to_string(getuid())}, - {"workDir", fs::current_path().string()}, - {"tmpDir", fs::current_path().string()} - }; + daggy::ConfigValues defaultJobValues{{"minCPUs", "1"}, + {"minMemoryMB", "100"}, + {"minTmpDiskMB", "0"}, + {"priority", "1"}, + {"timeLimitSeconds", "200"}, + {"userID", std::to_string(getuid())}, + {"workDir", fs::current_path().string()}, + {"tmpDir", fs::current_path().string()}}; - SECTION("Simple Run") { - daggy::Task task{.job{ - {"command", std::vector{"/usr/bin/echo", "abc", "123"}} - }}; + SECTION("Simple Run") + { + daggy::Task task{.job{ + {"command", std::vector{"/usr/bin/echo", "abc", "123"}}}}; - task.job.merge(defaultJobValues); + task.job.merge(defaultJobValues); - REQUIRE(ex.validateTaskParameters(task.job)); + REQUIRE(ex.validateTaskParameters(task.job)); - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); - REQUIRE(rec.rc == 0); - REQUIRE(rec.outputLog.size() >= 6); - REQUIRE(rec.errorLog.empty()); + REQUIRE(rec.rc == 0); + REQUIRE(rec.outputLog.size() >= 6); + REQUIRE(rec.errorLog.empty()); + } + + SECTION("Error Run") + { + daggy::Task task{ + .job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{ + "/usr/bin/expr", "1", "+", "+"}}}}; + task.job.merge(defaultJobValues); + + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); + + REQUIRE(rec.rc != 0); + REQUIRE(rec.errorLog.size() >= 20); + REQUIRE(rec.outputLog.empty()); + } + + SECTION("Large Output") + { + const std::vector BIG_FILES{"/usr/share/dict/linux.words", + "/usr/share/dict/cracklib-small", + "/etc/ssh/moduli"}; + + for (const auto &bigFile : BIG_FILES) { + if (!std::filesystem::exists(bigFile)) + continue; + + daggy::Task task{ + .job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{ + "/usr/bin/cat", bigFile}}}}; + task.job.merge(defaultJobValues); + + auto recFuture = ex.execute("command", task); + auto rec = recFuture.get(); + + REQUIRE(rec.rc == 0); + REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile)); + REQUIRE(rec.errorLog.empty()); + break; } + } - SECTION("Error Run") { - daggy::Task task{.job{ - {"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}}; - task.job.merge(defaultJobValues); + SECTION("Parameter Expansion") + { + std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; + auto params = daggy::configFromJSON(testParams); - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); + std::string taskJSON = + R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; + auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues); - REQUIRE(rec.rc != 0); - REQUIRE(rec.errorLog.size() >= 20); - REQUIRE(rec.outputLog.empty()); - } + auto result = daggy::expandTaskSet(tasks, ex, params); + REQUIRE(result.size() == 2); + } - SECTION("Large Output") { - const std::vector BIG_FILES{ - "/usr/share/dict/linux.words", "/usr/share/dict/cracklib-small", "/etc/ssh/moduli" - }; + SECTION("Build with expansion") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; + auto params = daggy::configFromJSON(testParams); + std::string testTasks = + R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; + auto tasks = daggy::expandTaskSet( + daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params); + REQUIRE(tasks.size() == 4); + } - for (const auto &bigFile: BIG_FILES) { - if (!std::filesystem::exists(bigFile)) continue; + SECTION("Build with expansion using parents instead of children") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; + auto params = daggy::configFromJSON(testParams); + std::string testTasks = + R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})"; + auto tasks = daggy::expandTaskSet( + daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params); - daggy::Task task{.job{ - {"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/cat", bigFile}}}}; - task.job.merge(defaultJobValues); - - auto recFuture = ex.execute("command", task); - auto rec = recFuture.get(); - - REQUIRE(rec.rc == 0); - REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile)); - REQUIRE(rec.errorLog.empty()); - break; - } - } - - SECTION("Parameter Expansion") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; - auto params = daggy::configFromJSON(testParams); - - std::string taskJSON = R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; - auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues); - - auto result = daggy::expandTaskSet(tasks, ex, params); - REQUIRE(result.size() == 2); - } - - SECTION("Build with expansion") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; - auto params = daggy::configFromJSON(testParams); - std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; - auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params); - REQUIRE(tasks.size() == 4); - } - - SECTION("Build with expansion using parents instead of children") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; - auto params = daggy::configFromJSON(testParams); - std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})"; - auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params); - - REQUIRE(tasks.size() == 4); - } + REQUIRE(tasks.size() == 4); + } } #endif diff --git a/tests/unit_serialization.cpp b/tests/unit_serialization.cpp index 75f1a4c..37b2109 100644 --- a/tests/unit_serialization.cpp +++ b/tests/unit_serialization.cpp @@ -1,35 +1,48 @@ -#include +#include #include #include - -#include +#include #include "daggy/Serialization.hpp" namespace fs = std::filesystem; -TEST_CASE("parameter_deserialization", "[deserialize_parameters]") { - SECTION("Basic Parse") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; - auto params = daggy::configFromJSON(testParams); - REQUIRE(params.size() == 2); - REQUIRE(std::holds_alternative>(params["DATE"])); - REQUIRE(std::holds_alternative(params["SOURCE"])); - }SECTION("Invalid JSON") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name")"}; - REQUIRE_THROWS(daggy::configFromJSON(testParams)); - }SECTION("Non-string Keys") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], 6: "name"})"}; - REQUIRE_THROWS(daggy::configFromJSON(testParams)); - }SECTION("Non-array/Non-string values") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": {"name": "kevin"}})"}; - REQUIRE_THROWS(daggy::configFromJSON(testParams)); - } +TEST_CASE("parameter_deserialization", "[deserialize_parameters]") +{ + SECTION("Basic Parse") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"}; + auto params = daggy::configFromJSON(testParams); + REQUIRE(params.size() == 2); + REQUIRE(std::holds_alternative>(params["DATE"])); + REQUIRE(std::holds_alternative(params["SOURCE"])); + } + SECTION("Invalid JSON") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name")"}; + REQUIRE_THROWS(daggy::configFromJSON(testParams)); + } + SECTION("Non-string Keys") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], 6: "name"})"}; + REQUIRE_THROWS(daggy::configFromJSON(testParams)); + } + SECTION("Non-array/Non-string values") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": {"name": "kevin"}})"}; + REQUIRE_THROWS(daggy::configFromJSON(testParams)); + } } -TEST_CASE("task_deserialization", "[deserialize_task]") { - SECTION("Build with no expansion") { - std::string testTasks = R"({ +TEST_CASE("task_deserialization", "[deserialize_task]") +{ + SECTION("Build with no expansion") + { + std::string testTasks = R"({ "A": { "job": { "command": ["/bin/echo", "A"] }, "children": ["C"] @@ -42,12 +55,13 @@ TEST_CASE("task_deserialization", "[deserialize_task]") { "job": {"command": ["/bin/echo", "C"]} } })"; - auto tasks = daggy::tasksFromJSON(testTasks); - REQUIRE(tasks.size() == 3); - } + auto tasks = daggy::tasksFromJSON(testTasks); + REQUIRE(tasks.size() == 3); + } - SECTION("Build with job defaults") { - std::string testTasks = R"({ + SECTION("Build with job defaults") + { + std::string testTasks = R"({ "A": { "job": { "command": ["/bin/echo", "A"] }, "children": ["B"] @@ -59,30 +73,32 @@ TEST_CASE("task_deserialization", "[deserialize_task]") { } } })"; - daggy::ConfigValues jobDefaults{{"runtime", "60"}, - {"memory", "300M"}}; - auto tasks = daggy::tasksFromJSON(testTasks, jobDefaults); - REQUIRE(tasks.size() == 2); - REQUIRE(std::get(tasks["A"].job["runtime"]) == "60"); - REQUIRE(std::get(tasks["A"].job["memory"]) == "300M"); - REQUIRE(std::get(tasks["B"].job["runtime"]) == "60"); - REQUIRE(std::get(tasks["B"].job["memory"]) == "1G"); - } + daggy::ConfigValues jobDefaults{{"runtime", "60"}, {"memory", "300M"}}; + auto tasks = daggy::tasksFromJSON(testTasks, jobDefaults); + REQUIRE(tasks.size() == 2); + REQUIRE(std::get(tasks["A"].job["runtime"]) == "60"); + REQUIRE(std::get(tasks["A"].job["memory"]) == "300M"); + REQUIRE(std::get(tasks["B"].job["runtime"]) == "60"); + REQUIRE(std::get(tasks["B"].job["memory"]) == "1G"); + } } -TEST_CASE("task_serialization", "[serialize_tasks]") { - SECTION("Build with no expansion") { - std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/echo", "B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; - auto tasks = daggy::tasksFromJSON(testTasks); +TEST_CASE("task_serialization", "[serialize_tasks]") +{ + SECTION("Build with no expansion") + { + std::string testTasks = + R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/echo", "B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})"; + auto tasks = daggy::tasksFromJSON(testTasks); - auto genJSON = daggy::tasksToJSON(tasks); - auto regenTasks = daggy::tasksFromJSON(genJSON); + auto genJSON = daggy::tasksToJSON(tasks); + auto regenTasks = daggy::tasksFromJSON(genJSON); - REQUIRE(regenTasks.size() == tasks.size()); + REQUIRE(regenTasks.size() == tasks.size()); - for (const auto &[name, task]: regenTasks) { - const auto &other = tasks[name]; - REQUIRE(task == other); - } + for (const auto &[name, task] : regenTasks) { + const auto &other = tasks[name]; + REQUIRE(task == other); } + } } diff --git a/tests/unit_server.cpp b/tests/unit_server.cpp index 09de8b6..98a1eaf 100644 --- a/tests/unit_server.cpp +++ b/tests/unit_server.cpp @@ -1,83 +1,85 @@ -#include -#include -#include - -#include #include #include -#include +#include #include +#include #include #include +#include +#include +#include namespace rj = rapidjson; -Pistache::Http::Response -REQUEST(std::string url, std::string payload = "") { - Pistache::Http::Experimental::Client client; - client.init(); - Pistache::Http::Response response; - auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); - reqSpec.timeout(std::chrono::seconds(2)); - if (!payload.empty()) { - reqSpec.body(payload); - } - auto request = reqSpec.send(); - bool ok = false, error = false; - std::string msg; - request.then( - [&](Pistache::Http::Response rsp) { - ok = true; - response = rsp; - }, - [&](std::exception_ptr ptr) { - error = true; - try { - std::rethrow_exception(ptr); - } catch (std::exception &e) { - msg = e.what(); - } - } - ); +Pistache::Http::Response REQUEST(std::string url, std::string payload = "") +{ + Pistache::Http::Experimental::Client client; + client.init(); + Pistache::Http::Response response; + auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); + reqSpec.timeout(std::chrono::seconds(2)); + if (!payload.empty()) { + reqSpec.body(payload); + } + auto request = reqSpec.send(); + bool ok = false, error = false; + std::string msg; + request.then( + [&](Pistache::Http::Response rsp) { + ok = true; + response = rsp; + }, + [&](std::exception_ptr ptr) { + error = true; + try { + std::rethrow_exception(ptr); + } + catch (std::exception &e) { + msg = e.what(); + } + }); - Pistache::Async::Barrier barrier(request); - barrier.wait_for(std::chrono::seconds(2)); - client.shutdown(); - if (error) { - throw std::runtime_error(msg); - } - return response; + Pistache::Async::Barrier barrier(request); + barrier.wait_for(std::chrono::seconds(2)); + client.shutdown(); + if (error) { + throw std::runtime_error(msg); + } + return response; } -TEST_CASE("rest_endpoint", "[server_basic]") { - std::stringstream ss; - daggy::executors::task::ForkingTaskExecutor executor(10); - daggy::loggers::dag_run::OStreamLogger logger(ss); - Pistache::Address listenSpec("localhost", Pistache::Port(0)); +TEST_CASE("rest_endpoint", "[server_basic]") +{ + std::stringstream ss; + daggy::executors::task::ForkingTaskExecutor executor(10); + daggy::loggers::dag_run::OStreamLogger logger(ss); + Pistache::Address listenSpec("localhost", Pistache::Port(0)); - const size_t nDAGRunners = 10, - nWebThreads = 10; + const size_t nDAGRunners = 10, nWebThreads = 10; - daggy::Server server(listenSpec, logger, executor, nDAGRunners); - server.init(nWebThreads); - server.start(); + daggy::Server server(listenSpec, logger, executor, nDAGRunners); + server.init(nWebThreads); + server.start(); - const std::string host = "localhost:"; - const std::string baseURL = host + std::to_string(server.getPort()); + const std::string host = "localhost:"; + const std::string baseURL = host + std::to_string(server.getPort()); - SECTION ("Ready Endpoint") { - auto response = REQUEST(baseURL + "/ready"); - REQUIRE(response.code() == Pistache::Http::Code::Ok); - } + SECTION("Ready Endpoint") + { + auto response = REQUEST(baseURL + "/ready"); + REQUIRE(response.code() == Pistache::Http::Code::Ok); + } - SECTION ("Querying a non-existent dagrunid should fail ") { - auto response = REQUEST(baseURL + "/v1/dagrun/100"); - REQUIRE(response.code() != Pistache::Http::Code::Ok); - } + SECTION("Querying a non-existent dagrunid should fail ") + { + auto response = REQUEST(baseURL + "/v1/dagrun/100"); + REQUIRE(response.code() != Pistache::Http::Code::Ok); + } - SECTION("Simple DAGRun Submission") { - std::string dagRun = R"({ + SECTION("Simple DAGRun Submission") + { + std::string dagRun = R"({ "name": "unit_server", "parameters": { "FILE": [ "A", "B" ] }, "tasks": { @@ -88,87 +90,89 @@ TEST_CASE("rest_endpoint", "[server_basic]") { } })"; + // Submit, and get the runID + daggy::DAGRunID runID = 0; + { + auto response = REQUEST(baseURL + "/v1/dagrun/", dagRun); + REQUIRE(response.code() == Pistache::Http::Code::Ok); - // Submit, and get the runID - daggy::DAGRunID runID = 0; - { - auto response = REQUEST(baseURL + "/v1/dagrun/", dagRun); - REQUIRE(response.code() == Pistache::Http::Code::Ok); + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body().c_str())); + REQUIRE(doc.IsObject()); + REQUIRE(doc.HasMember("runID")); - rj::Document doc; - daggy::checkRJParse(doc.Parse(response.body().c_str())); - REQUIRE(doc.IsObject()); - REQUIRE(doc.HasMember("runID")); - - runID = doc["runID"].GetUint64(); - } - - // Ensure our runID shows up in the list of running DAGs - { - auto response = REQUEST(baseURL + "/v1/dagrun/"); - REQUIRE(response.code() == Pistache::Http::Code::Ok); - - rj::Document doc; - daggy::checkRJParse(doc.Parse(response.body().c_str())); - REQUIRE(doc.IsArray()); - REQUIRE(doc.Size() >= 1); - - // Ensure that our DAG is in the list and matches our given DAGRunID - bool found = false; - const auto &runs = doc.GetArray(); - for (size_t i = 0; i < runs.Size(); ++i) { - const auto &run = runs[i]; - REQUIRE(run.IsObject()); - REQUIRE(run.HasMember("name")); - REQUIRE(run.HasMember("runID")); - - std::string runName = run["name"].GetString(); - if (runName == "unit_server") { - REQUIRE(run["runID"].GetUint64() == runID); - found = true; - break; - } - } - REQUIRE(found); - } - - // Wait until our DAG is complete - bool complete = true; - for (auto i = 0; i < 10; ++i) { - auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID)); - REQUIRE(response.code() == Pistache::Http::Code::Ok); - rj::Document doc; - daggy::checkRJParse(doc.Parse(response.body().c_str())); - REQUIRE(doc.IsObject()); - - REQUIRE(doc.HasMember("taskStates")); - const auto &taskStates = doc["taskStates"].GetObject(); - - size_t nStates = 0; - for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); ++it) { - nStates++; - } - REQUIRE(nStates == 3); - - complete = true; - for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); ++it) { - std::string state = it->value.GetString(); - if (state != "COMPLETED") { - complete = false; - break; - } - } - if (complete) break; - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - REQUIRE(complete); - - std::this_thread::sleep_for(std::chrono::seconds(2)); - for (const auto &pth: std::vector{"dagrun_A", "dagrun_B"}) { - REQUIRE(fs::exists(pth)); - fs::remove(pth); - } + runID = doc["runID"].GetUint64(); } - server.shutdown(); + // Ensure our runID shows up in the list of running DAGs + { + auto response = REQUEST(baseURL + "/v1/dagrun/"); + REQUIRE(response.code() == Pistache::Http::Code::Ok); + + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body().c_str())); + REQUIRE(doc.IsArray()); + REQUIRE(doc.Size() >= 1); + + // Ensure that our DAG is in the list and matches our given DAGRunID + bool found = false; + const auto &runs = doc.GetArray(); + for (size_t i = 0; i < runs.Size(); ++i) { + const auto &run = runs[i]; + REQUIRE(run.IsObject()); + REQUIRE(run.HasMember("name")); + REQUIRE(run.HasMember("runID")); + + std::string runName = run["name"].GetString(); + if (runName == "unit_server") { + REQUIRE(run["runID"].GetUint64() == runID); + found = true; + break; + } + } + REQUIRE(found); + } + + // Wait until our DAG is complete + bool complete = true; + for (auto i = 0; i < 10; ++i) { + auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID)); + REQUIRE(response.code() == Pistache::Http::Code::Ok); + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body().c_str())); + REQUIRE(doc.IsObject()); + + REQUIRE(doc.HasMember("taskStates")); + const auto &taskStates = doc["taskStates"].GetObject(); + + size_t nStates = 0; + for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); + ++it) { + nStates++; + } + REQUIRE(nStates == 3); + + complete = true; + for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); + ++it) { + std::string state = it->value.GetString(); + if (state != "COMPLETED") { + complete = false; + break; + } + } + if (complete) + break; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + REQUIRE(complete); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + for (const auto &pth : std::vector{"dagrun_A", "dagrun_B"}) { + REQUIRE(fs::exists(pth)); + fs::remove(pth); + } + } + + server.shutdown(); } diff --git a/tests/unit_threadpool.cpp b/tests/unit_threadpool.cpp index 9191a9a..507b673 100644 --- a/tests/unit_threadpool.cpp +++ b/tests/unit_threadpool.cpp @@ -1,41 +1,45 @@ -#include +#include #include +#include #include "daggy/ThreadPool.hpp" -#include - using namespace daggy; -TEST_CASE("threadpool", "[threadpool]") { - std::atomic cnt(0); - ThreadPool tp(10); +TEST_CASE("threadpool", "[threadpool]") +{ + std::atomic cnt(0); + ThreadPool tp(10); - std::vector> rets; + std::vector> rets; - SECTION("Adding large tasks queues with return values") { - auto tq = std::make_shared(); - std::vector> res; - for (size_t i = 0; i < 100; ++i) - res.emplace_back(std::move(tq->addTask([&cnt]() { - cnt++; - return cnt.load(); - }))); - tp.addTasks(tq); - for (auto &r: res) r.get(); - REQUIRE(cnt == 100); - } + SECTION("Adding large tasks queues with return values") + { + auto tq = std::make_shared(); + std::vector> res; + for (size_t i = 0; i < 100; ++i) + res.emplace_back(std::move(tq->addTask([&cnt]() { + cnt++; + return cnt.load(); + }))); + tp.addTasks(tq); + for (auto &r : res) + r.get(); + REQUIRE(cnt == 100); + } - SECTION("Slow runs") { - std::vector> res; - using namespace std::chrono_literals; - for (size_t i = 0; i < 100; ++i) - res.push_back(tp.addTask([&cnt]() { - std::this_thread::sleep_for(20ms); - cnt++; - return; - })); - for (auto &r: res) r.get(); - REQUIRE(cnt == 100); - } + SECTION("Slow runs") + { + std::vector> res; + using namespace std::chrono_literals; + for (size_t i = 0; i < 100; ++i) + res.push_back(tp.addTask([&cnt]() { + std::this_thread::sleep_for(20ms); + cnt++; + return; + })); + for (auto &r : res) + r.get(); + REQUIRE(cnt == 100); + } } diff --git a/tests/unit_utilities.cpp b/tests/unit_utilities.cpp index 8b0009f..73fd36d 100644 --- a/tests/unit_utilities.cpp +++ b/tests/unit_utilities.cpp @@ -1,74 +1,79 @@ -#include +#include +#include #include #include #include - #include -#include #include #include -#include - -#include "daggy/Utilities.hpp" #include "daggy/Serialization.hpp" +#include "daggy/Utilities.hpp" #include "daggy/executors/task/ForkingTaskExecutor.hpp" #include "daggy/executors/task/NoopTaskExecutor.hpp" #include "daggy/loggers/dag_run/OStreamLogger.hpp" namespace fs = std::filesystem; -TEST_CASE("string_utilities", "[utilities_string]") { - std::string test = "/this/is/{{A}}/test/{{A}}"; - auto res = daggy::globalSub(test, "{{A}}", "hello"); - REQUIRE(res == "/this/is/hello/test/hello"); +TEST_CASE("string_utilities", "[utilities_string]") +{ + std::string test = "/this/is/{{A}}/test/{{A}}"; + auto res = daggy::globalSub(test, "{{A}}", "hello"); + REQUIRE(res == "/this/is/hello/test/hello"); } -TEST_CASE("string_expansion", "[utilities_parameter_expansion]") { - SECTION("Basic expansion") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; - auto params = daggy::configFromJSON(testParams); - std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}", "{{TYPE}}"}; - auto allCommands = daggy::interpolateValues(cmd, params); - - REQUIRE(allCommands.size() == 6); - } - - SECTION("Skip over unused parameters") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; - auto params = daggy::configFromJSON(testParams); - std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"}; - auto allCommands = daggy::interpolateValues(cmd, params); - - // TYPE isn't used, so it's just |DATE| * |SOURCE| - REQUIRE(allCommands.size() == 2); - } - - SECTION("Expand within a command part") { - std::string testParams{ - R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"}; - auto params = daggy::configFromJSON(testParams); - std::vector cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"}; - auto result = daggy::interpolateValues(cmd, params); - - // TYPE isn't used, so it's just |DATE| * |SOURCE| - REQUIRE(result.size() == 4); - - - } -} - -TEST_CASE("dag_runner_order", "[dagrun_order]") { - daggy::executors::task::NoopTaskExecutor ex; - std::stringstream ss; - daggy::loggers::dag_run::OStreamLogger logger(ss); - - daggy::TimePoint startTime = daggy::Clock::now(); - - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07", "2021-05-08", "2021-05-09" ]})"}; +TEST_CASE("string_expansion", "[utilities_parameter_expansion]") +{ + SECTION("Basic expansion") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; auto params = daggy::configFromJSON(testParams); + std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}", + "{{TYPE}}"}; + auto allCommands = daggy::interpolateValues(cmd, params); - std::string taskJSON = R"({ + REQUIRE(allCommands.size() == 6); + } + + SECTION("Skip over unused parameters") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; + auto params = daggy::configFromJSON(testParams); + std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"}; + auto allCommands = daggy::interpolateValues(cmd, params); + + // TYPE isn't used, so it's just |DATE| * |SOURCE| + REQUIRE(allCommands.size() == 2); + } + + SECTION("Expand within a command part") + { + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"}; + auto params = daggy::configFromJSON(testParams); + std::vector cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"}; + auto result = daggy::interpolateValues(cmd, params); + + // TYPE isn't used, so it's just |DATE| * |SOURCE| + REQUIRE(result.size() == 4); + } +} + +TEST_CASE("dag_runner_order", "[dagrun_order]") +{ + daggy::executors::task::NoopTaskExecutor ex; + std::stringstream ss; + daggy::loggers::dag_run::OStreamLogger logger(ss); + + daggy::TimePoint startTime = daggy::Clock::now(); + + std::string testParams{ + R"({"DATE": ["2021-05-06", "2021-05-07", "2021-05-08", "2021-05-09" ]})"}; + auto params = daggy::configFromJSON(testParams); + + std::string taskJSON = R"({ "A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]}, "B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]}, "C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]}, @@ -76,160 +81,175 @@ TEST_CASE("dag_runner_order", "[dagrun_order]") { "E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}} })"; - auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex, params); + auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex, params); - REQUIRE(tasks.size() == 20); + REQUIRE(tasks.size() == 20); - auto dag = daggy::buildDAGFromTasks(tasks); - auto runID = logger.startDAGRun("test_run", tasks); + auto dag = daggy::buildDAGFromTasks(tasks); + auto runID = logger.startDAGRun("test_run", tasks); + auto endDAG = daggy::runDAG(runID, ex, logger, dag); + + REQUIRE(endDAG.allVisited()); + + // Ensure the run order + auto rec = logger.getDAGRun(runID); + + daggy::TimePoint stopTime = daggy::Clock::now(); + std::array minTimes; + minTimes.fill(startTime); + std::array maxTimes; + maxTimes.fill(stopTime); + + for (const auto &[k, v] : rec.taskAttempts) { + size_t idx = k[0] - 65; + auto &startTime = minTimes[idx]; + auto &stopTime = maxTimes[idx]; + startTime = std::max(startTime, v.front().startTime); + stopTime = std::min(stopTime, v.back().stopTime); + } + + for (size_t i = 0; i < 5; ++i) { + for (size_t j = i + 1; j < 4; ++j) { + REQUIRE(maxTimes[i] < minTimes[j]); + } + } +} + +TEST_CASE("dag_runner", "[utilities_dag_runner]") +{ + daggy::executors::task::ForkingTaskExecutor ex(10); + std::stringstream ss; + daggy::loggers::dag_run::OStreamLogger logger(ss); + + SECTION("Simple execution") + { + std::string prefix = (fs::current_path() / "asdlk").string(); + std::unordered_map files{ + {"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}}; + std::string taskJSON = + R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + files.at("A") + + R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" + + files.at("B") + + R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" + + files.at("C") + R"("]}}})"; + auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); + auto dag = daggy::buildDAGFromTasks(tasks); + auto runID = logger.startDAGRun("test_run", tasks); auto endDAG = daggy::runDAG(runID, ex, logger, dag); REQUIRE(endDAG.allVisited()); - // Ensure the run order - auto rec = logger.getDAGRun(runID); - - daggy::TimePoint stopTime = daggy::Clock::now(); - std::array minTimes; minTimes.fill(startTime); - std::array maxTimes; maxTimes.fill(stopTime); - - for (const auto &[k, v] : rec.taskAttempts) { - size_t idx = k[0] - 65; - auto & startTime = minTimes[idx]; - auto & stopTime = maxTimes[idx]; - startTime = std::max(startTime, v.front().startTime); - stopTime = std::min(stopTime, v.back().stopTime); + for (const auto &[_, file] : files) { + REQUIRE(fs::exists(file)); + fs::remove(file); } - for (size_t i = 0; i < 5; ++i) { - for (size_t j = i+1; j < 4; ++j) { - REQUIRE(maxTimes[i] < minTimes[j]); - } - } -} - -TEST_CASE("dag_runner", "[utilities_dag_runner]") { - daggy::executors::task::ForkingTaskExecutor ex(10); - std::stringstream ss; - daggy::loggers::dag_run::OStreamLogger logger(ss); - - SECTION("Simple execution") { - std::string prefix = (fs::current_path() / "asdlk").string(); - std::unordered_map files{ - {"A", prefix + "_A"}, - {"B", prefix + "_B"}, - {"C", prefix + "_C"}}; - std::string taskJSON = R"({"A": {"job": {"command": ["/usr/bin/touch", ")" - + files.at("A") + R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" - + files.at("B") + R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" - + files.at("C") + R"("]}}})"; - auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); - auto dag = daggy::buildDAGFromTasks(tasks); - auto runID = logger.startDAGRun("test_run", tasks); - auto endDAG = daggy::runDAG(runID, ex, logger, dag); - - REQUIRE(endDAG.allVisited()); - - for (const auto &[_, file] : files) { - REQUIRE(fs::exists(file)); - fs::remove(file); - } - - // Get the DAG Run Attempts - auto record = logger.getDAGRun(runID); - for (const auto &[_, attempts]: record.taskAttempts) { - REQUIRE(attempts.size() == 1); - REQUIRE(attempts.front().rc == 0); - } - } - - SECTION("Recovery from Error") { - auto cleanup = []() { - // Cleanup - std::vector paths{"rec_error_A", "noexist"}; - for (const auto &pth: paths) { - if (fs::exists(pth)) fs::remove_all(pth); - } - }; - - cleanup(); - - std::string goodPrefix = "rec_error_"; - std::string badPrefix = "noexist/rec_error_"; - std::string taskJSON = R"({"A": {"job": {"command": ["/usr/bin/touch", ")" - + goodPrefix + - R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" - + badPrefix + - R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" - + badPrefix + R"(C"]}}})"; - auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); - auto dag = daggy::buildDAGFromTasks(tasks); - - auto runID = logger.startDAGRun("test_run", tasks); - - auto tryDAG = daggy::runDAG(runID, ex, logger, dag); - - REQUIRE(!tryDAG.allVisited()); - - // Create the missing dir, then continue to run the DAG - fs::create_directory("noexist"); - tryDAG.resetRunning(); - auto endDAG = daggy::runDAG(runID, ex, logger, tryDAG); - - REQUIRE(endDAG.allVisited()); - - // Get the DAG Run Attempts - auto record = logger.getDAGRun(runID); - REQUIRE(record.taskAttempts["A_0"].size() == 1); // A ran fine - REQUIRE(record.taskAttempts["B_0"].size() == 2); // B errored and had to be retried - REQUIRE(record.taskAttempts["C_0"].size() == 1); // C wasn't run because B errored - - cleanup(); - } - - SECTION("Generator tasks") { - std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; - auto params = daggy::configFromJSON(testParams); - - std::string generatorOutput = R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})"; - fs::path ofn = fs::current_path() / "generator_test_output.json"; - std::ofstream ofh{ofn}; - ofh << generatorOutput << std::endl; - ofh.close(); - - std::stringstream jsonTasks; - - jsonTasks << R"({ "A": { "job": {"command": [ "/usr/bin/cat", )" << std::quoted(ofn.string()) - << R"(]}, "children": ["C"], "isGenerator": true},)" - << R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })"; - - auto baseTasks = daggy::tasksFromJSON(jsonTasks.str()); - REQUIRE(baseTasks.size() == 2); - auto tasks = daggy::expandTaskSet(baseTasks, ex, params); - REQUIRE(tasks.size() == 2); - auto dag = daggy::buildDAGFromTasks(tasks); - REQUIRE(dag.size() == 2); - - auto runID = logger.startDAGRun("generator_run", tasks); - auto finalDAG = daggy::runDAG(runID, ex, logger, dag, params); - - REQUIRE(finalDAG.allVisited()); - REQUIRE(finalDAG.size() == 4); - - // Check the logger - auto record = logger.getDAGRun(runID); - - REQUIRE(record.tasks.size() == 4); - REQUIRE(record.taskRunStates.size() == 4); - for (const auto &[taskName, attempts]: record.taskAttempts) { - REQUIRE(attempts.size() == 1); - REQUIRE(attempts.back().rc == 0); - } - - // Ensure that children were updated properly - REQUIRE(record.tasks["A_0"].children == std::unordered_set{"B_0", "B_1", "C"}); - REQUIRE(record.tasks["B_0"].children == std::unordered_set{"C"}); - REQUIRE(record.tasks["B_1"].children == std::unordered_set{"C"}); - REQUIRE(record.tasks["C_0"].children.empty()); + // Get the DAG Run Attempts + auto record = logger.getDAGRun(runID); + for (const auto &[_, attempts] : record.taskAttempts) { + REQUIRE(attempts.size() == 1); + REQUIRE(attempts.front().rc == 0); } + } + + SECTION("Recovery from Error") + { + auto cleanup = []() { + // Cleanup + std::vector paths{"rec_error_A", "noexist"}; + for (const auto &pth : paths) { + if (fs::exists(pth)) + fs::remove_all(pth); + } + }; + + cleanup(); + + std::string goodPrefix = "rec_error_"; + std::string badPrefix = "noexist/rec_error_"; + std::string taskJSON = + R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + goodPrefix + + R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" + + badPrefix + + R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" + + badPrefix + R"(C"]}}})"; + auto tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); + auto dag = daggy::buildDAGFromTasks(tasks); + + auto runID = logger.startDAGRun("test_run", tasks); + + auto tryDAG = daggy::runDAG(runID, ex, logger, dag); + + REQUIRE(!tryDAG.allVisited()); + + // Create the missing dir, then continue to run the DAG + fs::create_directory("noexist"); + tryDAG.resetRunning(); + auto endDAG = daggy::runDAG(runID, ex, logger, tryDAG); + + REQUIRE(endDAG.allVisited()); + + // Get the DAG Run Attempts + auto record = logger.getDAGRun(runID); + REQUIRE(record.taskAttempts["A_0"].size() == 1); // A ran fine + REQUIRE(record.taskAttempts["B_0"].size() == + 2); // B errored and had to be retried + REQUIRE(record.taskAttempts["C_0"].size() == + 1); // C wasn't run because B errored + + cleanup(); + } + + SECTION("Generator tasks") + { + std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"}; + auto params = daggy::configFromJSON(testParams); + + std::string generatorOutput = + R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})"; + fs::path ofn = fs::current_path() / "generator_test_output.json"; + std::ofstream ofh{ofn}; + ofh << generatorOutput << std::endl; + ofh.close(); + + std::stringstream jsonTasks; + + jsonTasks + << R"({ "A": { "job": {"command": [ "/usr/bin/cat", )" + << std::quoted(ofn.string()) + << R"(]}, "children": ["C"], "isGenerator": true},)" + << R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })"; + + auto baseTasks = daggy::tasksFromJSON(jsonTasks.str()); + REQUIRE(baseTasks.size() == 2); + auto tasks = daggy::expandTaskSet(baseTasks, ex, params); + REQUIRE(tasks.size() == 2); + auto dag = daggy::buildDAGFromTasks(tasks); + REQUIRE(dag.size() == 2); + + auto runID = logger.startDAGRun("generator_run", tasks); + auto finalDAG = daggy::runDAG(runID, ex, logger, dag, params); + + REQUIRE(finalDAG.allVisited()); + REQUIRE(finalDAG.size() == 4); + + // Check the logger + auto record = logger.getDAGRun(runID); + + REQUIRE(record.tasks.size() == 4); + REQUIRE(record.taskRunStates.size() == 4); + for (const auto &[taskName, attempts] : record.taskAttempts) { + REQUIRE(attempts.size() == 1); + REQUIRE(attempts.back().rc == 0); + } + + // Ensure that children were updated properly + REQUIRE(record.tasks["A_0"].children == + std::unordered_set{"B_0", "B_1", "C"}); + REQUIRE(record.tasks["B_0"].children == + std::unordered_set{"C"}); + REQUIRE(record.tasks["B_1"].children == + std::unordered_set{"C"}); + REQUIRE(record.tasks["C_0"].children.empty()); + } } diff --git a/utils/daggyc/daggyc.cpp b/utils/daggyc/daggyc.cpp index 9e03f8a..d1aa75a 100644 --- a/utils/daggyc/daggyc.cpp +++ b/utils/daggyc/daggyc.cpp @@ -1,78 +1,77 @@ -#include -#include -#include - -#include - #include #include +#include +#include +#include +#include + namespace rj = rapidjson; -Pistache::Http::Response -REQUEST(std::string url, std::string payload = "") { - Pistache::Http::Experimental::Client client; - client.init(); - Pistache::Http::Response response; - auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); - reqSpec.timeout(std::chrono::seconds(2)); - if (!payload.empty()) { - reqSpec.body(payload); - } - auto request = reqSpec.send(); - bool ok = false, error = false; - std::string msg; - request.then( - [&](Pistache::Http::Response rsp) { - ok = true; - response = rsp; - }, - [&](std::exception_ptr ptr) { - error = true; - try { - std::rethrow_exception(ptr); - } catch (std::exception &e) { - msg = e.what(); - } - } - ); +Pistache::Http::Response REQUEST(std::string url, std::string payload = "") +{ + Pistache::Http::Experimental::Client client; + client.init(); + Pistache::Http::Response response; + auto reqSpec = (payload.empty() ? client.get(url) : client.post(url)); + reqSpec.timeout(std::chrono::seconds(2)); + if (!payload.empty()) { + reqSpec.body(payload); + } + auto request = reqSpec.send(); + bool ok = false, error = false; + std::string msg; + request.then( + [&](Pistache::Http::Response rsp) { + ok = true; + response = rsp; + }, + [&](std::exception_ptr ptr) { + error = true; + try { + std::rethrow_exception(ptr); + } + catch (std::exception &e) { + msg = e.what(); + } + }); - Pistache::Async::Barrier barrier(request); - barrier.wait_for(std::chrono::seconds(2)); - client.shutdown(); - if (error) { - throw std::runtime_error(msg); - } - return response; + Pistache::Async::Barrier barrier(request); + barrier.wait_for(std::chrono::seconds(2)); + client.shutdown(); + if (error) { + throw std::runtime_error(msg); + } + return response; } -int main(int argc, char **argv) { - argparse::ArgumentParser args("Daggy Client"); +int main(int argc, char **argv) +{ + argparse::ArgumentParser args("Daggy Client"); - args.add_argument("-v", "--verbose") - .default_value(false) - .implicit_value(true); - args.add_argument("--url") - .help("base URL of server") - .default_value("http://localhost:2503"); - args.add_argument("--sync") - .default_value(false) - .implicit_value(true) - .help("Poll for job to complete"); - args.add_argument("--action") - .help("Number of tasks to run concurrently") - .default_value(30) - .action([](const std::string &value) { return std::stoull(value); }); + args.add_argument("-v", "--verbose") + .default_value(false) + .implicit_value(true); + args.add_argument("--url") + .help("base URL of server") + .default_value("http://localhost:2503"); + args.add_argument("--sync").default_value(false).implicit_value(true).help( + "Poll for job to complete"); + args.add_argument("--action") + .help("Number of tasks to run concurrently") + .default_value(30) + .action([](const std::string &value) { return std::stoull(value); }); - try { - args.parse_args(argc, argv); - } catch (std::exception &e) { - std::cout << "Error: " << e.what() << std::endl; - std::cout << args; - exit(1); - } + try { + args.parse_args(argc, argv); + } + catch (std::exception &e) { + std::cout << "Error: " << e.what() << std::endl; + std::cout << args; + exit(1); + } - std::string baseURL = args.get("--url"); + std::string baseURL = args.get("--url"); - auto response = REQUEST(baseURL + "/ready"); + auto response = REQUEST(baseURL + "/ready"); } diff --git a/utils/daggyd/daggyd.cpp b/utils/daggyd/daggyd.cpp index d2d6fb3..c7bb6c6 100644 --- a/utils/daggyd/daggyd.cpp +++ b/utils/daggyd/daggyd.cpp @@ -1,13 +1,11 @@ -#include -#include -#include - -#include #include +#include #include - +#include #include +#include +#include // Add executors here #ifdef DAGGY_ENABLE_SLURM @@ -24,182 +22,198 @@ /* #include #include -#include #include #include +#include */ static std::atomic running{true}; -void signalHandler(int signal) { - switch (signal) { - case SIGHUP: - break; - case SIGINT: - case SIGTERM: - running = false; - break; - } +void signalHandler(int signal) +{ + switch (signal) { + case SIGHUP: + break; + case SIGINT: + case SIGTERM: + running = false; + break; + } } -void daemonize() { - pid_t pid; +void daemonize() +{ + pid_t pid; - struct sigaction newSigAction; - sigset_t newSigSet; + struct sigaction newSigAction; + sigset_t newSigSet; - /* Check if parent process id is set */ - if (getppid() == 1) { return; } + /* Check if parent process id is set */ + if (getppid() == 1) { + return; + } - /* Set signal mask - signals we want to block */ - sigemptyset(&newSigSet); - sigaddset(&newSigSet, SIGCHLD); /* ignore child - i.e. we don't need to wait for it */ - sigaddset(&newSigSet, SIGTSTP); /* ignore Tty stop signals */ - sigaddset(&newSigSet, SIGTTOU); /* ignore Tty background writes */ - sigaddset(&newSigSet, SIGTTIN); /* ignore Tty background reads */ - sigprocmask(SIG_BLOCK, &newSigSet, NULL); /* Block the above specified signals */ + /* Set signal mask - signals we want to block */ + sigemptyset(&newSigSet); + sigaddset(&newSigSet, + SIGCHLD); /* ignore child - i.e. we don't need to wait for it */ + sigaddset(&newSigSet, SIGTSTP); /* ignore Tty stop signals */ + sigaddset(&newSigSet, SIGTTOU); /* ignore Tty background writes */ + sigaddset(&newSigSet, SIGTTIN); /* ignore Tty background reads */ + sigprocmask(SIG_BLOCK, &newSigSet, + NULL); /* Block the above specified signals */ - /* Set up a signal handler */ - newSigAction.sa_handler = signalHandler; - sigemptyset(&newSigAction.sa_mask); - newSigAction.sa_flags = 0; + /* Set up a signal handler */ + newSigAction.sa_handler = signalHandler; + sigemptyset(&newSigAction.sa_mask); + newSigAction.sa_flags = 0; - /* Signals to handle */ - sigaction(SIGHUP, &newSigAction, NULL); /* catch hangup signal */ - sigaction(SIGTERM, &newSigAction, NULL); /* catch term signal */ - sigaction(SIGINT, &newSigAction, NULL); /* catch interrupt signal */ + /* Signals to handle */ + sigaction(SIGHUP, &newSigAction, NULL); /* catch hangup signal */ + sigaction(SIGTERM, &newSigAction, NULL); /* catch term signal */ + sigaction(SIGINT, &newSigAction, NULL); /* catch interrupt signal */ - // Fork once - pid = fork(); - if (pid < 0) { exit(EXIT_FAILURE); } - if (pid > 0) { exit(EXIT_SUCCESS); } + // Fork once + pid = fork(); + if (pid < 0) { + exit(EXIT_FAILURE); + } + if (pid > 0) { + exit(EXIT_SUCCESS); + } - /* On success: The child process becomes session leader */ - if (setsid() < 0) { - std::cerr << "Unable to setsid" << std::endl; - exit(EXIT_FAILURE); - } + /* On success: The child process becomes session leader */ + if (setsid() < 0) { + std::cerr << "Unable to setsid" << std::endl; + exit(EXIT_FAILURE); + } - /* Catch, ignore and handle signals */ - signal(SIGCHLD, SIG_IGN); - signal(SIGHUP, SIG_IGN); + /* Catch, ignore and handle signals */ + signal(SIGCHLD, SIG_IGN); + signal(SIGHUP, SIG_IGN); - /* Fork off for the second time*/ - pid = fork(); - if (pid < 0) - exit(EXIT_FAILURE); - if (pid > 0) - exit(EXIT_SUCCESS); + /* Fork off for the second time*/ + pid = fork(); + if (pid < 0) + exit(EXIT_FAILURE); + if (pid > 0) + exit(EXIT_SUCCESS); - umask(0); + umask(0); - /* Change the working directory to the root directory */ - /* or another appropriated directory */ - auto rc = chdir("/"); - (void)rc; + /* Change the working directory to the root directory */ + /* or another appropriated directory */ + auto rc = chdir("/"); + (void)rc; - /* Close all open file descriptors */ - for (auto x = sysconf(_SC_OPEN_MAX); x >= 0; x--) { close(x); } + /* Close all open file descriptors */ + for (auto x = sysconf(_SC_OPEN_MAX); x >= 0; x--) { + close(x); + } } -int main(int argc, char **argv) { - argparse::ArgumentParser args("Daggy"); +int main(int argc, char **argv) +{ + argparse::ArgumentParser args("Daggy"); - args.add_argument("-v", "--verbose") - .default_value(false) - .implicit_value(true); - args.add_argument("-d", "--daemon") - .default_value(false) - .implicit_value(true); - args.add_argument("--ip") - .help("IP address to listen to") - .default_value(std::string{"127.0.0.1"}); - args.add_argument("--log-file") - .help("File to log to.") - .default_value(std::string{"daggyd.log"}); - args.add_argument("--port") - .help("Port to listen to") - .default_value(2503) - .action([](const std::string &value) { return std::stoi(value); }); - args.add_argument("--dag-threads") - .help("Number of DAGs to run concurrently") - .default_value(10UL) - .action([](const std::string &value) { return std::stoull(value); }); - args.add_argument("--web-threads") - .help("Number of web requests to support concurrently") - .default_value(30UL) - .action([](const std::string &value) { return std::stoull(value); }); - args.add_argument("--executor-threads") - .help("Number of tasks to run concurrently") - .default_value(30UL) - .action([](const std::string &value) { return std::stoull(value); }); + args.add_argument("-v", "--verbose") + .default_value(false) + .implicit_value(true); + args.add_argument("-d", "--daemon").default_value(false).implicit_value(true); + args.add_argument("--ip") + .help("IP address to listen to") + .default_value(std::string{"127.0.0.1"}); + args.add_argument("--log-file") + .help("File to log to.") + .default_value(std::string{"daggyd.log"}); + args.add_argument("--port") + .help("Port to listen to") + .default_value(2503) + .action([](const std::string &value) { return std::stoi(value); }); + args.add_argument("--dag-threads") + .help("Number of DAGs to run concurrently") + .default_value(10UL) + .action([](const std::string &value) { return std::stoull(value); }); + args.add_argument("--web-threads") + .help("Number of web requests to support concurrently") + .default_value(30UL) + .action([](const std::string &value) { return std::stoull(value); }); + args.add_argument("--executor-threads") + .help("Number of tasks to run concurrently") + .default_value(30UL) + .action([](const std::string &value) { return std::stoull(value); }); - try { - args.parse_args(argc, argv); - } catch (std::exception &e) { - std::cout << "Error: " << e.what() << std::endl; - std::cout << args; - exit(1); - } + try { + args.parse_args(argc, argv); + } + catch (std::exception &e) { + std::cout << "Error: " << e.what() << std::endl; + std::cout << args; + exit(1); + } - bool verbose = args.get("--verbose"); - bool asDaemon = args.get("--daemon"); - std::string logFileName = args.get("--log-file"); - std::string listenIP = args.get("--ip"); - uint16_t listenPort = args.get("--port"); - size_t executorThreads = args.get("--executor-threads"); - size_t webThreads = args.get("--web-threads"); - size_t dagThreads = args.get("--dag-threads"); - - if (logFileName == "-") { - if (asDaemon) { - std::cout << "Unable to daemonize if logging to stdout" << std::endl; - exit(1); - } - } else { - fs::path logFn{logFileName}; - if (!logFn.is_absolute()) { - logFileName = (fs::current_path() / logFileName).string(); - } - } - - if (verbose) { - std::cout << "Server running at http://" << listenIP << ':' << listenPort << std::endl - << "Max DAG Processing: " << dagThreads << std::endl - << "Max Task Execution: " << executorThreads << std::endl - << "Max Web Clients: " << webThreads << std::endl - << "Logging to: " << logFileName << std::endl - << std::endl << "Ctrl-C to exit" << std::endl; - } + bool verbose = args.get("--verbose"); + bool asDaemon = args.get("--daemon"); + std::string logFileName = args.get("--log-file"); + std::string listenIP = args.get("--ip"); + uint16_t listenPort = args.get("--port"); + size_t executorThreads = args.get("--executor-threads"); + size_t webThreads = args.get("--web-threads"); + size_t dagThreads = args.get("--dag-threads"); + if (logFileName == "-") { if (asDaemon) { - daemonize(); + std::cout << "Unable to daemonize if logging to stdout" << std::endl; + exit(1); } + } + else { + fs::path logFn{logFileName}; + if (!logFn.is_absolute()) { + logFileName = (fs::current_path() / logFileName).string(); + } + } - std::ofstream logFH; - std::unique_ptr logger; - if (logFileName == "-") { - logger = std::make_unique(std::cout); - } else { - logFH.open(logFileName, std::ios::app); - logger = std::make_unique(logFH); - } + if (verbose) { + std::cout << "Server running at http://" << listenIP << ':' << listenPort + << std::endl + << "Max DAG Processing: " << dagThreads << std::endl + << "Max Task Execution: " << executorThreads << std::endl + << "Max Web Clients: " << webThreads << std::endl + << "Logging to: " << logFileName << std::endl + << std::endl + << "Ctrl-C to exit" << std::endl; + } + + if (asDaemon) { + daemonize(); + } + + std::ofstream logFH; + std::unique_ptr logger; + if (logFileName == "-") { + logger = + std::make_unique(std::cout); + } + else { + logFH.open(logFileName, std::ios::app); + logger = std::make_unique(logFH); + } #ifdef DAGGY_ENABLE_SLURM - daggy::executors::task::SlurmTaskExecutor executor; + daggy::executors::task::SlurmTaskExecutor executor; #else - daggy::executors::task::ForkingTaskExecutor executor(executorThreads); + daggy::executors::task::ForkingTaskExecutor executor(executorThreads); #endif - Pistache::Address listenSpec(listenIP, listenPort); + Pistache::Address listenSpec(listenIP, listenPort); - daggy::Server server(listenSpec, *logger, executor, dagThreads); - server.init(webThreads); - server.start(); + daggy::Server server(listenSpec, *logger, executor, dagThreads); + server.init(webThreads); + server.start(); - - running = true; - while (running) { - std::this_thread::sleep_for(std::chrono::seconds(30)); - } - server.shutdown(); + running = true; + while (running) { + std::this_thread::sleep_for(std::chrono::seconds(30)); + } + server.shutdown(); }