Adding support for remote execution daemons.

Squashed commit of the following:

commit 69d5ef7a256b86a86d46e5ae374c00fded1497ea
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 12:15:55 2021 -0400

    Updating readme

commit 94a9f676d0f9cc0b55cdc18c4927eaea40d82c77
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 12:05:36 2021 -0400

    Fixing serialization of attempt records when querying entire dag

commit 945e5f90b24abf07c9af1bc4c6bbcb33e93b8069
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 11:37:59 2021 -0400

    Compiles cleanly...

commit 8b23e46081d47fb80dc1a2d998fc6dc4bbf301a8
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 10:43:03 2021 -0400

    Adding in missing source file to cmake build list

commit 6d10d9791206e2bc15788beadeea580b8e43a853
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 10:41:43 2021 -0400

    Adding new executors

commit 42a2c67f4d6ae99df95d917c8621d78cd99837a1
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 10:27:14 2021 -0400

    Fixing missing curl cmake dependency

commit 394bc4c5d51ecee7bf14712f719c8bf7e97fb0fa
Author: Ian Roddis <tech@kinesin.ca>
Date:   Thu Dec 16 10:21:58 2021 -0400

    Fixing missing curl cmake dependency

commit dd9efc8e7e7770ea1bcbccb70a1af9cfcff0414c
Author: Ian Roddis <tech@kinesin.ca>
Date:   Wed Dec 15 17:15:38 2021 -0400

    Checkpointing progress

commit 3b3b55d6037bb96e46de6763f486f4ecb92fe6a0
Author: Ian Roddis <tech@kinesin.ca>
Date:   Wed Dec 15 14:21:18 2021 -0400

    updating readme

commit 303027c11452941b2a0c0d1b04ac5942e79efd74
Author: Ian Roddis <tech@kinesin.ca>
Date:   Wed Dec 15 14:17:16 2021 -0400

    Namespacing daggyd
    Adding more error checking around deserialization of parameters
    Adding tests for runner agent

commit c592eaeba12e2a449bae401e8c1d9ed236416d52
Author: Ian Roddis <tech@kinesin.ca>
Date:   Wed Dec 15 11:20:21 2021 -0400

    Checkpointing work

commit fb1862d1cefe2b53a98659cce3c8c73d88bf5d84
Author: Ian Roddis <tech@kinesin.ca>
Date:   Wed Dec 15 09:52:29 2021 -0400

    Copying daggyd for daggyr template, adding in basic routes
This commit is contained in:
Ian Roddis
2021-12-16 12:16:12 -04:00
parent 14d0ef4a3f
commit 8d00621908
26 changed files with 1373 additions and 160 deletions

View File

@@ -1,5 +1,6 @@
#pragma once
#include <curl/curl.h>
#include <rapidjson/document.h>
#include <string>
@@ -12,6 +13,8 @@
#include "daggy/executors/task/TaskExecutor.hpp"
#include "daggy/loggers/dag_run/DAGRunLogger.hpp"
namespace rj = rapidjson;
namespace daggy {
using TaskDAG = DAG<std::string, Task>;
@@ -40,4 +43,48 @@ namespace daggy {
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks);
std::ostream &operator<<(std::ostream &os, const TimePoint &tp);
// HTTP helpers
enum HTTPCode : long
{
Ok = 200,
Not_Found = 404,
Not_Acceptable = 406
};
struct HTTPResponse
{
HTTPCode code;
std::string body;
};
HTTPResponse HTTP_REQUEST(const std::string &url,
const std::string &payload = "",
const std::string &method = "GET",
bool trace = false);
std::pair<HTTPCode, rj::Document> JSON_HTTP_REQUEST(
const std::string &url, const std::string &payload = "",
const std::string &method = "GET", bool trace = false);
} // namespace daggy
template <typename T>
void hash_combine(std::size_t &seed, T const &key)
{
std::hash<T> hasher;
seed ^= hasher(key) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}
namespace std {
template <typename T1, typename T2>
struct hash<std::pair<T1, T2>>
{
std::size_t operator()(std::pair<T1, T2> const &p) const
{
std::size_t seed(0);
::hash_combine(seed, p.first);
::hash_combine(seed, p.second);
return seed;
}
};
} // namespace std

View File

@@ -0,0 +1,69 @@
#pragma once
#include <rapidjson/document.h>
#include "TaskExecutor.hpp"
namespace rj = rapidjson;
namespace daggy::executors::task {
namespace daggy_runner {
struct Capacity
{
ssize_t cores;
ssize_t memoryMB;
};
std::string capacityToJSON(const Capacity &cap);
Capacity capacityFromJSON(const rj::Value &spec);
Capacity capacityFromTask(const Task &task);
void validateTaskParameters(const ConfigValues &job);
} // namespace daggy_runner
class DaggyRunnerTaskExecutor : public TaskExecutor
{
public:
using Command = std::vector<std::string>;
DaggyRunnerTaskExecutor();
~DaggyRunnerTaskExecutor() override;
// Validates the job to ensure that all required values are set and are of
// the right type,
bool validateTaskParameters(const ConfigValues &job) override;
std::vector<ConfigValues> expandTaskParameters(
const ConfigValues &job, const ConfigValues &expansionValues) override;
// Runs the task
std::future<AttemptRecord> execute(DAGRunID runID,
const std::string &taskName,
const Task &task) override;
bool stop(DAGRunID runID, const std::string &taskName) override;
void addRunner(const std::string &url);
private:
void monitor();
struct RunningTask
{
std::promise<AttemptRecord> prom;
DAGRunID runID;
std::string taskName;
std::string runnerURL;
};
// Resolves jobs through polling
std::atomic<bool> running_;
std::thread monitorWorker_;
std::unordered_set<std::string> runners_;
std::mutex rtGuard_;
std::unordered_map<std::pair<DAGRunID, std::string>, RunningTask>
runningTasks_;
};
} // namespace daggy::executors::task

View File

@@ -5,6 +5,10 @@
#include "TaskExecutor.hpp"
namespace daggy::executors::task {
namespace forking_executor {
void validateTaskParameters(const ConfigValues &job);
}
class ForkingTaskExecutor : public TaskExecutor
{
public: