Adding support for remote execution daemons.
Squashed commit of the following: commit 69d5ef7a256b86a86d46e5ae374c00fded1497ea Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 12:15:55 2021 -0400 Updating readme commit 94a9f676d0f9cc0b55cdc18c4927eaea40d82c77 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 12:05:36 2021 -0400 Fixing serialization of attempt records when querying entire dag commit 945e5f90b24abf07c9af1bc4c6bbcb33e93b8069 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 11:37:59 2021 -0400 Compiles cleanly... commit 8b23e46081d47fb80dc1a2d998fc6dc4bbf301a8 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:43:03 2021 -0400 Adding in missing source file to cmake build list commit 6d10d9791206e2bc15788beadeea580b8e43a853 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:41:43 2021 -0400 Adding new executors commit 42a2c67f4d6ae99df95d917c8621d78cd99837a1 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:27:14 2021 -0400 Fixing missing curl cmake dependency commit 394bc4c5d51ecee7bf14712f719c8bf7e97fb0fa Author: Ian Roddis <tech@kinesin.ca> Date: Thu Dec 16 10:21:58 2021 -0400 Fixing missing curl cmake dependency commit dd9efc8e7e7770ea1bcbccb70a1af9cfcff0414c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 17:15:38 2021 -0400 Checkpointing progress commit 3b3b55d6037bb96e46de6763f486f4ecb92fe6a0 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 14:21:18 2021 -0400 updating readme commit 303027c11452941b2a0c0d1b04ac5942e79efd74 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 14:17:16 2021 -0400 Namespacing daggyd Adding more error checking around deserialization of parameters Adding tests for runner agent commit c592eaeba12e2a449bae401e8c1d9ed236416d52 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 11:20:21 2021 -0400 Checkpointing work commit fb1862d1cefe2b53a98659cce3c8c73d88bf5d84 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Dec 15 09:52:29 2021 -0400 Copying daggyd for daggyr template, adding in basic routes
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
#pragma once
|
||||
|
||||
#include <rapidjson/document.h>
|
||||
|
||||
#include "TaskExecutor.hpp"
|
||||
|
||||
namespace rj = rapidjson;
|
||||
|
||||
namespace daggy::executors::task {
|
||||
|
||||
namespace daggy_runner {
|
||||
struct Capacity
|
||||
{
|
||||
ssize_t cores;
|
||||
ssize_t memoryMB;
|
||||
};
|
||||
|
||||
std::string capacityToJSON(const Capacity &cap);
|
||||
Capacity capacityFromJSON(const rj::Value &spec);
|
||||
Capacity capacityFromTask(const Task &task);
|
||||
|
||||
void validateTaskParameters(const ConfigValues &job);
|
||||
} // namespace daggy_runner
|
||||
|
||||
class DaggyRunnerTaskExecutor : public TaskExecutor
|
||||
{
|
||||
public:
|
||||
using Command = std::vector<std::string>;
|
||||
|
||||
DaggyRunnerTaskExecutor();
|
||||
~DaggyRunnerTaskExecutor() override;
|
||||
|
||||
// Validates the job to ensure that all required values are set and are of
|
||||
// the right type,
|
||||
bool validateTaskParameters(const ConfigValues &job) override;
|
||||
|
||||
std::vector<ConfigValues> expandTaskParameters(
|
||||
const ConfigValues &job, const ConfigValues &expansionValues) override;
|
||||
|
||||
// Runs the task
|
||||
std::future<AttemptRecord> execute(DAGRunID runID,
|
||||
const std::string &taskName,
|
||||
const Task &task) override;
|
||||
|
||||
bool stop(DAGRunID runID, const std::string &taskName) override;
|
||||
|
||||
void addRunner(const std::string &url);
|
||||
|
||||
private:
|
||||
void monitor();
|
||||
|
||||
struct RunningTask
|
||||
{
|
||||
std::promise<AttemptRecord> prom;
|
||||
DAGRunID runID;
|
||||
std::string taskName;
|
||||
std::string runnerURL;
|
||||
};
|
||||
|
||||
// Resolves jobs through polling
|
||||
std::atomic<bool> running_;
|
||||
std::thread monitorWorker_;
|
||||
|
||||
std::unordered_set<std::string> runners_;
|
||||
std::mutex rtGuard_;
|
||||
std::unordered_map<std::pair<DAGRunID, std::string>, RunningTask>
|
||||
runningTasks_;
|
||||
};
|
||||
} // namespace daggy::executors::task
|
||||
@@ -5,6 +5,10 @@
|
||||
#include "TaskExecutor.hpp"
|
||||
|
||||
namespace daggy::executors::task {
|
||||
namespace forking_executor {
|
||||
void validateTaskParameters(const ConfigValues &job);
|
||||
}
|
||||
|
||||
class ForkingTaskExecutor : public TaskExecutor
|
||||
{
|
||||
public:
|
||||
|
||||
Reference in New Issue
Block a user