Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

checkpointing

Browse Source
This commit is contained in:
Ian Roddis
2021-06-13 09:29:48 -03:00
parent 9cfa8cab7f
commit 2cfe3e21d5
4 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
daggy/include/daggy/Executor.hpp
View File

@@ -1,10 +1,10 @@
#pragma once #pragma once
#include <vector> #include <chrono>
#include <thread>
#include <future> #include <future>
#include <string> #include <string>
#include <chrono> #include <thread>
#include <vector>
#include "Task.hpp" #include "Task.hpp"

8
daggy/include/daggy/Scheduler.hpp
View File

@@ -1,21 +1,25 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include <unordered_map>
#include <string> #include <string>
#include <unordered_map>
#include <variant>
#include "DAG.hpp" #include "DAG.hpp"
#include "Executor.hpp" #include "Executor.hpp"
namespace daggy { namespace daggy {
using Parameter = std::variant<std::string, std::vector<std::string>>;
class Scheduler { class Scheduler {
public: public:
// Register an executor // Register an executor
void registerExecutor(std::shared_ptr<Executor> executor); void registerExecutor(std::shared_ptr<Executor> executor);
void runDAG(DAG dag); void runTasks(std::vector<Task> tasks, std::vector<Parameter> parameters);
private: private:
std::unordered_map<std::string, std::shared_ptr<Executor>> executors; std::unordered_map<std::string, std::shared_ptr<Executor>> executors;
std::unordered_map<std::string, std::vector<std::future<TaskResult>>> jobs; std::unordered_map<std::string, std::vector<std::future<TaskResult>>> jobs;
std::vector<Parameter> parameters;
}; };
} }

4
daggy/include/daggy/Task.hpp
View File

@@ -1,7 +1,8 @@
#pragma once #pragma once
#include <string>
#include <cstdint> #include <cstdint>
#include <string>
#include <vector>
namespace daggy { namespace daggy {
struct Task { struct Task {
@@ -9,5 +10,6 @@ namespace daggy {
std::string command; std::string command;
uint8_t max_retries; uint8_t max_retries;
uint32_t retry_interval_seconds; // Time to wait between retries uint32_t retry_interval_seconds; // Time to wait between retries
std::vector<std::string> children;
}; };
} }

2
examples/sample_dag.json
View File

@@ -15,6 +15,7 @@
"command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_A", "command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_A",
"verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_A", "verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_A",
"timeout_seconds": 30 "timeout_seconds": 30
"children": [ "merge_data" ]
}, },
{ {
"name": "pull_data_b", "name": "pull_data_b",
@@ -23,6 +24,7 @@
"command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_B", "command": "/path/to/pull.sh --date {DATE} --source {SOURCE}_B",
"verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_B", "verification_command": "/path/to/pull_verify.sh --date {DATE} --source {SOURCE}_B",
"timeout_seconds": 30 "timeout_seconds": 30
"children": [ "merge_data" ]
}, },
{ {
"name": "merge_data", "name": "merge_data",