checkpointing

daggy/include/daggy/Executor.hpp

@@ -1,10 +1,10 @@
 #pragma once
 
-#include <vector>
-#include <thread>
+#include <chrono>
 #include <future>
 #include <string>
-#include <chrono>
+#include <thread>
+#include <vector>
 
 #include "Task.hpp"
 

daggy/include/daggy/Scheduler.hpp

@@ -1,21 +1,25 @@
 #pragma once
 
 #include <memory>
-#include <unordered_map>
 #include <string>
+#include <unordered_map>
+#include <variant>
 
 #include "DAG.hpp"
 #include "Executor.hpp"
 
 namespace daggy {
+  using Parameter = std::variant<std::string, std::vector<std::string>>;
+
   class Scheduler {
     public:
       // Register an executor
       void registerExecutor(std::shared_ptr<Executor> executor);
-      void runDAG(DAG dag);
+      void runTasks(std::vector<Task> tasks, std::vector<Parameter> parameters);
 
     private:
       std::unordered_map<std::string, std::shared_ptr<Executor>> executors;
       std::unordered_map<std::string, std::vector<std::future<TaskResult>>> jobs;
+      std::vector<Parameter> parameters;
   };
 }

daggy/include/daggy/Task.hpp

@@ -1,7 +1,8 @@
 #pragma once
 
-#include <string>
 #include <cstdint>
+#include <string>
+#include <vector>
 
 namespace daggy {
   struct Task {
@@ -9,5 +10,6 @@ namespace daggy {
     std::string command;
     uint8_t     max_retries;
     uint32_t    retry_interval_seconds; // Time to wait between retries
+    std::vector<std::string> children;
   };
 }

examples/sample_dag.json

@@ -15,6 +15,7 @@
       "command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_A",
       "verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_A",
       "timeout_seconds": 30
+      "children": [ "merge_data" ]
     },
     {
       "name": "pull_data_b",
@@ -23,6 +24,7 @@
       "command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_B",
       "verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_B",
       "timeout_seconds": 30
+      "children": [ "merge_data" ]
     },
     {
       "name": "merge_data",