Adding support for remote execution daemons.
Squashed commit of the following: commit 69d5ef7a256b86a86d46e5ae374c00fded1497ea Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 12:15:55 2021 -0400 Updating readme commit 94a9f676d0f9cc0b55cdc18c4927eaea40d82c77 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 12:05:36 2021 -0400 Fixing serialization of attempt records when querying entire dag commit 945e5f90b24abf07c9af1bc4c6bbcb33e93b8069 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 11:37:59 2021 -0400 Compiles cleanly... commit 8b23e46081d47fb80dc1a2d998fc6dc4bbf301a8 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:43:03 2021 -0400 Adding in missing source file to cmake build list commit 6d10d9791206e2bc15788beadeea580b8e43a853 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:41:43 2021 -0400 Adding new executors commit 42a2c67f4d6ae99df95d917c8621d78cd99837a1 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:27:14 2021 -0400 Fixing missing curl cmake dependency commit 394bc4c5d51ecee7bf14712f719c8bf7e97fb0fa Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:21:58 2021 -0400 Fixing missing curl cmake dependency commit dd9efc8e7e7770ea1bcbccb70a1af9cfcff0414c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 17:15:38 2021 -0400 Checkpointing progress commit 3b3b55d6037bb96e46de6763f486f4ecb92fe6a0 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 14:21:18 2021 -0400 updating readme commit 303027c11452941b2a0c0d1b04ac5942e79efd74 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 14:17:16 2021 -0400 Namespacing daggyd Adding more error checking around deserialization of parameters Adding tests for runner agent commit c592eaeba12e2a449bae401e8c1d9ed236416d52 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 11:20:21 2021 -0400 Checkpointing work commit fb1862d1cefe2b53a98659cce3c8c73d88bf5d84 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 09:52:29 2021 -0400 Copying daggyd for daggyr template, adding in basic routes
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <rapidjson/document.h>
|
||||
|
||||
#include <string>
|
||||
@@ -12,6 +13,8 @@
|
||||
#include "daggy/executors/task/TaskExecutor.hpp"
|
||||
#include "daggy/loggers/dag_run/DAGRunLogger.hpp"
|
||||
|
||||
namespace rj = rapidjson;
|
||||
|
||||
namespace daggy {
|
||||
using TaskDAG = DAG<std::string, Task>;
|
||||
|
||||
@@ -40,4 +43,48 @@ namespace daggy {
|
||||
void updateDAGFromTasks(TaskDAG &dag, const TaskSet &tasks);
|
||||
|
||||
std::ostream &operator<<(std::ostream &os, const TimePoint &tp);
|
||||
|
||||
// HTTP helpers
|
||||
enum HTTPCode : long
|
||||
{
|
||||
Ok = 200,
|
||||
Not_Found = 404,
|
||||
Not_Acceptable = 406
|
||||
};
|
||||
|
||||
struct HTTPResponse
|
||||
{
|
||||
HTTPCode code;
|
||||
std::string body;
|
||||
};
|
||||
|
||||
HTTPResponse HTTP_REQUEST(const std::string &url,
|
||||
const std::string &payload = "",
|
||||
const std::string &method = "GET",
|
||||
bool trace = false);
|
||||
|
||||
std::pair<HTTPCode, rj::Document> JSON_HTTP_REQUEST(
|
||||
const std::string &url, const std::string &payload = "",
|
||||
const std::string &method = "GET", bool trace = false);
|
||||
} // namespace daggy
|
||||
|
||||
template <typename T>
|
||||
void hash_combine(std::size_t &seed, T const &key)
|
||||
{
|
||||
std::hash<T> hasher;
|
||||
seed ^= hasher(key) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
||||
}
|
||||
|
||||
namespace std {
|
||||
template <typename T1, typename T2>
|
||||
struct hash<std::pair<T1, T2>>
|
||||
{
|
||||
std::size_t operator()(std::pair<T1, T2> const &p) const
|
||||
{
|
||||
std::size_t seed(0);
|
||||
::hash_combine(seed, p.first);
|
||||
::hash_combine(seed, p.second);
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
} // namespace std
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#pragma once
|
||||
|
||||
#include <rapidjson/document.h>
|
||||
|
||||
#include "TaskExecutor.hpp"
|
||||
|
||||
namespace rj = rapidjson;
|
||||
|
||||
namespace daggy::executors::task {
|
||||
|
||||
namespace daggy_runner {
|
||||
struct Capacity
|
||||
{
|
||||
ssize_t cores;
|
||||
ssize_t memoryMB;
|
||||
};
|
||||
|
||||
std::string capacityToJSON(const Capacity &cap);
|
||||
Capacity capacityFromJSON(const rj::Value &spec);
|
||||
Capacity capacityFromTask(const Task &task);
|
||||
|
||||
void validateTaskParameters(const ConfigValues &job);
|
||||
} // namespace daggy_runner
|
||||
|
||||
class DaggyRunnerTaskExecutor : public TaskExecutor
|
||||
{
|
||||
public:
|
||||
using Command = std::vector<std::string>;
|
||||
|
||||
DaggyRunnerTaskExecutor();
|
||||
~DaggyRunnerTaskExecutor() override;
|
||||
|
||||
// Validates the job to ensure that all required values are set and are of
|
||||
// the right type,
|
||||
bool validateTaskParameters(const ConfigValues &job) override;
|
||||
|
||||
std::vector<ConfigValues> expandTaskParameters(
|
||||
const ConfigValues &job, const ConfigValues &expansionValues) override;
|
||||
|
||||
// Runs the task
|
||||
std::future<AttemptRecord> execute(DAGRunID runID,
|
||||
const std::string &taskName,
|
||||
const Task &task) override;
|
||||
|
||||
bool stop(DAGRunID runID, const std::string &taskName) override;
|
||||
|
||||
void addRunner(const std::string &url);
|
||||
|
||||
private:
|
||||
void monitor();
|
||||
|
||||
struct RunningTask
|
||||
{
|
||||
std::promise<AttemptRecord> prom;
|
||||
DAGRunID runID;
|
||||
std::string taskName;
|
||||
std::string runnerURL;
|
||||
};
|
||||
|
||||
// Resolves jobs through polling
|
||||
std::atomic<bool> running_;
|
||||
std::thread monitorWorker_;
|
||||
|
||||
std::unordered_set<std::string> runners_;
|
||||
std::mutex rtGuard_;
|
||||
std::unordered_map<std::pair<DAGRunID, std::string>, RunningTask>
|
||||
runningTasks_;
|
||||
};
|
||||
} // namespace daggy::executors::task
|
||||
@@ -5,6 +5,10 @@
|
||||
#include "TaskExecutor.hpp"
|
||||
|
||||
namespace daggy::executors::task {
|
||||
namespace forking_executor {
|
||||
void validateTaskParameters(const ConfigValues &job);
|
||||
}
|
||||
|
||||
class ForkingTaskExecutor : public TaskExecutor
|
||||
{
|
||||
public:
|
||||
|
||||
Reference in New Issue
Block a user