diff --git a/daggy/CMakeLists.txt b/daggy/CMakeLists.txt index ddee947..15bd0b2 100644 --- a/daggy/CMakeLists.txt +++ b/daggy/CMakeLists.txt @@ -2,7 +2,7 @@ project(daggy) #ExternalProject_Add_StepDependencies(pistache_extern build) -file(GLOB SOURCES src/*.cpp src/**/*.cpp) +file(GLOB_RECURSE SOURCES src/*.cpp) add_library(${PROJECT_NAME} STATIC ${SOURCES}) include_directories(${PISTACHE_INCLUDE_DIR}) target_include_directories(${PROJECT_NAME} PUBLIC include) diff --git a/daggy/include/daggy/Utilities.hpp b/daggy/include/daggy/Utilities.hpp index 747f9a7..0acd89b 100644 --- a/daggy/include/daggy/Utilities.hpp +++ b/daggy/include/daggy/Utilities.hpp @@ -7,7 +7,7 @@ #include -#include "daggy/loggers/dag_run/DAGLoggerBase.hpp" +#include "daggy/loggers/dag_run/DAGLogger.hpp" #include "daggy/executors/task/TaskExecutor.hpp" #include "Task.hpp" #include "Defines.hpp" @@ -24,12 +24,12 @@ namespace daggy { TaskID taskID, const Task &task, executors::task::TaskExecutor &executor, - loggers::dag_run::DAGLoggerBase &logger); + loggers::dag_run::DAGLogger &logger); void runDAG(DAGRunID runID, std::vector tasks, executors::task::TaskExecutor &executor, - loggers::dag_run::DAGLoggerBase &logger, + loggers::dag_run::DAGLogger &logger, DAG dag); } diff --git a/daggy/include/daggy/loggers/dag_run/DAGLogger.hpp b/daggy/include/daggy/loggers/dag_run/DAGLogger.hpp new file mode 100644 index 0000000..2029bbc --- /dev/null +++ b/daggy/include/daggy/loggers/dag_run/DAGLogger.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include "../../Task.hpp" +#include "../../AttemptRecord.hpp" +#include "../../Defines.hpp" +#include "Defines.hpp" + +/* + DAGLogger represents the interface to store all the state information + for daggy to run. Abstracted in case other back-end solutions need to + be supported. +*/ + +namespace daggy { + namespace loggers { + namespace dag_run { + class DAGLogger { + public: + // Execution + virtual DAGRunID startDAGRun(std::string name, const std::vector &tasks) = 0; + + virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) = 0; + + virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) = 0; + + virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) = 0; + + // Querying + virtual std::vector getDAGs(uint32_t stateMask) = 0; + + virtual DAGRunRecord getDAGRun(DAGRunID dagRunId) = 0; + }; + } + } +} diff --git a/daggy/include/daggy/loggers/dag_run/DAGLoggerBase.hpp b/daggy/include/daggy/loggers/dag_run/DAGLoggerBase.hpp deleted file mode 100644 index 17decd4..0000000 --- a/daggy/include/daggy/loggers/dag_run/DAGLoggerBase.hpp +++ /dev/null @@ -1,73 +0,0 @@ -#pragma once - -#include - -#include "daggy/DAGRun.hpp" - -/* - DAGLoggerBase represents the interface to store all the state information - for daggy to run. Abstracted in case other back-end solutions need to - be supported. -*/ - -namespace daggy { - namespace loggers { - namespace dag_run { - enum class RunState : uint32_t { - QUEUED = 0, - RUNNING = 1, - RETRY = 1 << 1, - ERRORED = 1 << 2, - KILLED = 1 << 3, - COMPLETED = 1 << 4 - }; - - struct TaskUpdateRecord { - TimePoint time; - TaskID taskID; - RunState newState; - }; - - struct DAGUpdateRecord { - TimePoint time; - RunState newState; - }; - - // Pretty heavy weight, but - struct DAGRunRecord { - std::string name; - std::vector tasks; - std::vector runStates; - std::vector> taskAttempts; - std::vector taskStateChanges; - std::vector dagStateChanges; - }; - - struct DAGRunSummary { - DAGRunID runID; - std::string name; - RunState runState; - TimePoint startTime; - TimePoint lastUpdate; - std::unordered_map taskStateCounts; - }; - - class DAGLoggerBase { - public: - // Execution - virtual DAGRunID startDAGRun(std::string name, const std::vector &tasks) = 0; - - virtual void updateDAGRunState(DAGRunID dagRunId, RunState state) = 0; - - virtual void logTaskAttempt(DAGRunID, TaskID taskID, const AttemptRecord &attempt) = 0; - - virtual void updateTaskState(DAGRunID dagRunId, TaskID taskID, RunState state) = 0; - - // Querying - virtual std::vector getDAGs(uint32_t stateMask) = 0; - - virtual DAGRunRecord getDAGRun(DAGRunID dagRunId) = 0; - }; - } - } -} diff --git a/daggy/include/daggy/loggers/dag_run/Defines.hpp b/daggy/include/daggy/loggers/dag_run/Defines.hpp new file mode 100644 index 0000000..0b9112e --- /dev/null +++ b/daggy/include/daggy/loggers/dag_run/Defines.hpp @@ -0,0 +1,46 @@ +#pragma once + +namespace daggy { + namespace loggers { + namespace dag_run { + enum class RunState : uint32_t { + QUEUED = 0, + RUNNING = 1, + RETRY = 1 << 1, + ERRORED = 1 << 2, + KILLED = 1 << 3, + COMPLETED = 1 << 4 + }; + + struct TaskUpdateRecord { + TimePoint time; + TaskID taskID; + RunState newState; + }; + + struct DAGUpdateRecord { + TimePoint time; + RunState newState; + }; + + // Pretty heavy weight, but + struct DAGRunRecord { + std::string name; + std::vector tasks; + std::vector runStates; + std::vector > taskAttempts; + std::vector taskStateChanges; + std::vector dagStateChanges; + }; + + struct DAGRunSummary { + DAGRunID runID; + std::string name; + RunState runState; + TimePoint startTime; + TimePoint lastUpdate; + std::unordered_map taskStateCounts; + }; + } + } +} \ No newline at end of file diff --git a/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp b/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp index 0884b71..955b821 100644 --- a/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp +++ b/daggy/include/daggy/loggers/dag_run/FileSystemLogger.hpp @@ -5,7 +5,7 @@ #include #include -#include "DAGLoggerBase.hpp" +#include "DAGLogger.hpp" namespace fs = std::filesystem; namespace rj = rapidjson; @@ -34,7 +34,7 @@ namespace daggy { * stderr * execlog */ - class FileSystemLogger : DAGLoggerBase { + class FileSystemLogger : public DAGLogger { public: FileSystemLogger(fs::path root); diff --git a/daggy/include/daggy/loggers/dag_run/StdOutLogger.hpp b/daggy/include/daggy/loggers/dag_run/StdOutLogger.hpp index a0b6815..ceb3bc5 100644 --- a/daggy/include/daggy/loggers/dag_run/StdOutLogger.hpp +++ b/daggy/include/daggy/loggers/dag_run/StdOutLogger.hpp @@ -3,7 +3,7 @@ #include #include -#include "DAGLoggerBase.hpp" +#include "DAGLogger.hpp" namespace daggy { namespace loggers { @@ -12,7 +12,7 @@ namespace daggy { * This logger should only be used for debug purposes. It doesn't actually log anything, just prints stuff * to stdout. */ - class StdOutLogger : public DAGLoggerBase { + class StdOutLogger : public DAGLogger { public: StdOutLogger(); diff --git a/daggy/src/Utilities.cpp b/daggy/src/Utilities.cpp index b65a8d4..72ffb4c 100644 --- a/daggy/src/Utilities.cpp +++ b/daggy/src/Utilities.cpp @@ -53,7 +53,7 @@ namespace daggy { TaskID taskID, const Task &task, executors::task::TaskExecutor &executor, - loggers::dag_run::DAGLoggerBase &logger) { + loggers::dag_run::DAGLogger &logger) { std::vector attempts; logger.updateTaskState(runID, taskID, loggers::dag_run::RunState::RUNNING); @@ -69,7 +69,7 @@ namespace daggy { void runDAG(DAGRunID runID, std::vector tasks, executors::task::TaskExecutor &executor, - loggers::dag_run::DAGLoggerBase &logger, + loggers::dag_run::DAGLogger &logger, DAG dag) { logger.updateDAGRunState(runID, loggers::dag_run::RunState::RUNNING);