Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

* Formatting code with clang-tidy

Browse Source 
* Roughing in more metastore work
This commit is contained in:
Ian Roddis
2021-07-22 12:57:51 -03:00
parent 987718334f
commit 0349a5109b
16 changed files with 561 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
daggy/include/daggy/AttemptRecord.hpp Normal file
View File

@@ -0,0 +1,17 @@
#pragma once
#include <chrono>
#include <string>
namespace daggy {
using Clock = std::chrono::system_clock;
struct AttemptRecord {
std::chrono::time_point<Clock> startTime;
std::chrono::time_point<Clock> stopTime;
int rc; // RC from the task
std::string metaLog; // Logs from the executor
std::string output; // stdout from command
std::string error; // stderr from command
};
}

14
daggy/include/daggy/DAG.hpp
View File

@@ -28,30 +28,38 @@ namespace daggy {
std::unordered_set<size_t> children; std::unordered_set<size_t> children;
}; };
using Edge = std::pair<size_t,size_t>; using Edge = std::pair<size_t, size_t>;
class DAG { class DAG {
public: public:
// Vertices // Vertices
size_t addVertex(); size_t addVertex();
const std::vector<Vertex> & getVertices();
const std::vector<Vertex> &getVertices();
// Edges // Edges
void addEdge(const size_t src, const size_t dst); void addEdge(const size_t src, const size_t dst);
void dropEdge(const size_t src, const size_t dst); void dropEdge(const size_t src, const size_t dst);
bool hasPath(const size_t from, const size_t to) const; bool hasPath(const size_t from, const size_t to) const;
const std::vector<Edge> & getEdges();
const std::vector<Edge> &getEdges();
// Attributes // Attributes
size_t size() const; size_t size() const;
bool empty() const; bool empty() const;
// Traversal // Traversal
void reset(); void reset();
VertexState getVertexState(const size_t id) const; VertexState getVertexState(const size_t id) const;
bool allVisited() const; bool allVisited() const;
std::optional<const size_t> visitNext(); std::optional<const size_t> visitNext();
void completeVisit(const size_t id); void completeVisit(const size_t id);
private: private:

70
daggy/include/daggy/DAGImpl.hpp
View File

@@ -1,70 +0,0 @@
size_t DAG::size() const { return vertices_.size(); }
bool DAG::empty() const { return vertices_.empty(); }
size_t DAG::addVertex() {
vertices_.push_back(Vertex{.state = VertexState::UNVISITED, .depCount = 0});
return vertices_.size();
}
void DAG::dropEdge(const size_t from, const size_t to) {
vertices_[from].children.extract(to);
}
void DAG::addEdge(const size_t from, const size_t to) {
if (hasPath(to, from))
throw std::runtime_error("Adding edge would result in a cycle");
vertices_[from].children.insert(to);
}
bool DAG::hasPath(const size_t from, const size_t to) const {
bool pathFound = false;
for (const auto & child : vertices_[from].children) {
if (child == to) return true;
if (hasPath(child, to)) return true;
}
return false;
}
void DAG::reset() {
// Reset the state of all vertices
for (auto & v : vertices_) {
v.state = VertexState::UNVISITED;
v.depCount = 0;
}
// Calculate the upstream count
for (auto & v : vertices_) {
for (auto c : v.children) {
++vertices_[c].depCount;
}
}
}
bool DAG::allVisited() const {
for (const auto & v : vertices_) {
if (v.state != VertexState::VISITED) return false;
}
return true;
}
std::optional<const size_t > DAG::visitNext() {
for (size_t i = 0; i < vertices_.size(); ++i) {
auto & v = vertices_[i];
if (v.state != VertexState::UNVISITED) continue;
if (v.depCount != 0) continue;
v.state = VertexState::VISITING;
return i;
}
return {};
}
void DAG::completeVisit(const size_t id) {
auto & v = vertices_[id];
v.state = VertexState::VISITED;
for (auto c : v.children) {
--vertices_[c].depCount;
}
}

21
daggy/include/daggy/DAGRun.hpp Normal file
View File

@@ -0,0 +1,21 @@
#pragma once
#include <string>
#include <unordered_map>
#include <variant>
#include "DAG.hpp"
#include "Task.hpp"
#include "AttemptRecord.hpp"
namespace daggy {
using ParameterValue = std::variant<std::string, std::vector<std::string>>;
using TaskRun = std::vector<AttemptRecord>;
struct DAGRun {
std::vector<Task> tasks;
std::unordered_map<std::string, ParameterValue> parameters;
DAG dag;
std::vector<TaskRun> taskRuns;
};
}

14
daggy/include/daggy/Executor.hpp
View File

@@ -7,7 +7,7 @@
#include <vector> #include <vector>
#include "Task.hpp" #include "Task.hpp"
#include "AttemptRecord.hpp"
/* /*
Executors run Tasks, returning a future with the results. Executors run Tasks, returning a future with the results.
@@ -15,20 +15,10 @@
*/ */
namespace daggy { namespace daggy {
using Clock = std::chrono::system_clock;
struct AttemptRecord {
std::chrono::time_point<Clock> startTime;
std::chrono::time_point<Clock> stopTime;
int rc; // RC from the task
std::string metaLog; // Logs from the executor
std::string output; // stdout from command
std::string error; // stderr from command
};
class Executor { class Executor {
public: public:
Executor() = default; Executor() = default;
virtual const std::string getName() const = 0; virtual const std::string getName() const = 0;
// This will block if the executor is full // This will block if the executor is full

26
daggy/include/daggy/MetaStore.hpp
View File

@@ -2,6 +2,8 @@
#include <string> #include <string>
#include "DAGRun.hpp"
/* /*
MetaStore represents the interface to store all the state information MetaStore represents the interface to store all the state information
for daggy to run. Abstracted in case other back-end solutions need to for daggy to run. Abstracted in case other back-end solutions need to
@@ -9,17 +11,31 @@
*/ */
namespace daggy { namespace daggy {
using DAGDefID = int16_t; // future proofing using DAGDefID = int16_t;
using DAGRunID = size_t;
// This struct will contain transitions for
struct DAGRunEvent { };
class MetaStore { class MetaStore {
// Basic storage + retrieval of DAG Definitions // Basic storage + retrieval of DAG Definitions
virtual void storeDAGDefinition(std::string name, std::string definition) = 0; virtual DAGDefID storeDAGDefinition(std::string name, std::string definition) = 0;
virtual DAGDefID getCurrentDAGVersion(std::string name) = 0; virtual DAGDefID getCurrentDAGVersion(std::string name) = 0;
virtual std::string getDAGDefinition(std::string name, DAGDefID version = -1) = 0; virtual std::string getDAGDefinition(std::string name, DAGDefID version = -1) = 0;
// DAG Run State // DAG Run State
/*
* startDAGRun // DAG starts up, returns a DAGID for future updates
* updateDAGRun // DAG State transitions
* updateTaskState // Task state updates
*/
virtual DAGRunID startDAGRun(std::string dagName, DAGDefID version, DAGRun dagRun
) = 0;
virtual void updateTask(DAGRunID rid, std::string taskName, VertexState state) = 0;
virtual void updateDAGRun(DAGRunID rid, DAGState state) = 0;
// Retrievals
}; };
} }

36
daggy/include/daggy/Scheduler.hpp
View File

@@ -7,14 +7,10 @@
#include "DAG.hpp" #include "DAG.hpp"
#include "Executor.hpp" #include "Executor.hpp"
#include "DAGRun.hpp"
#include "ThreadPool.hpp" #include "ThreadPool.hpp"
namespace daggy { namespace daggy {
using ParameterValue = std::variant<std::string, std::vector<std::string>>;
using TaskRun = std::vector<AttemptRecord>;
class Scheduler {
public:
enum class DAGState : uint32_t { enum class DAGState : uint32_t {
UNKNOWN = 0, UNKNOWN = 0,
QUEUED, QUEUED,
@@ -23,25 +19,21 @@ namespace daggy {
COMPLETE COMPLETE
}; };
class Scheduler {
public:
public: public:
Scheduler( Scheduler(
Executor & executor Executor &executor, size_t executorThreads = 30, size_t schedulerThreads = 10);
, size_t executorThreads = 30
, size_t schedulerThreads = 10);
~Scheduler(); ~Scheduler();
// returns DagRun ID // returns DagRun ID
std::future<void> std::future<void>
scheduleDAG(std::string runName scheduleDAG(std::string runName, std::vector<Task> tasks,
, std::vector<Task> tasks std::unordered_map<std::string, ParameterValue> parameters,
, std::unordered_map<std::string, ParameterValue> parameters DAG dag = {} // Allows for loading of an existing DAG
, DAG dag = {} // Allows for loading of an existing DAG
); );
// get the current status of a DAG
DAGState dagRunStatus(std::string runName);
// get the current DAG // get the current DAG
DAG dagRunState(); DAG dagRunState();
@@ -49,21 +41,13 @@ namespace daggy {
void drain(); void drain();
private: private:
void runDAG(const std::string &name, DAGRun &dagRun);
struct DAGRun { std::vector<AttemptRecord> runTask(const Task &task);
std::vector<Task> tasks;
std::unordered_map<std::string, ParameterValue> parameters;
DAG dag;
std::vector<TaskRun> taskRuns;
std::mutex taskGuard_;
};
void runDAG(const std::string & name, DAGRun & dagRun);
std::vector<AttemptRecord> runTask(const Task & task);
std::unordered_map<std::string, DAGRun> runs_; std::unordered_map<std::string, DAGRun> runs_;
std::vector<std::future<void>> futs_; std::vector<std::future<void>> futs_;
Executor & executor_; Executor &executor_;
ThreadPool schedulers_; ThreadPool schedulers_;
ThreadPool executors_; ThreadPool executors_;
std::unordered_map<std::string, std::future<void>> jobs; std::unordered_map<std::string, std::future<void>> jobs;

21
daggy/include/daggy/Server.hpp
View File

@@ -10,9 +10,7 @@ namespace daggy {
class Server { class Server {
public: public:
Server(Pistache::Address addr) Server(Pistache::Address addr)
: endpoint_(addr) : endpoint_(addr), desc_("Daggy API", "0.1") {}
, desc_("Daggy API", "0.1")
{}
void init(int threads = 1); void init(int threads = 1);
@@ -25,22 +23,25 @@ namespace daggy {
// DAG Definition handlers // DAG Definition handlers
// //
void listDAGs(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void listDAGs(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
void upsertDAG(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response);
void deleteDAG(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void upsertDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
void getDAG(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response);
void deleteDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
void getDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
// //
// DAG Runs // DAG Runs
// //
void runDAG(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void runDAG(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
// List // List
void getDAGRuns(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void getDAGRuns(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
// Get status of specific run // Get status of specific run
void getDAGRun(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void getDAGRun(const Pistache::Rest::Request &request, Pistache::Http::ResponseWriter response);
Pistache::Http::Endpoint endpoint_; Pistache::Http::Endpoint endpoint_;
Pistache::Rest::Description desc_; Pistache::Rest::Description desc_;

25
daggy/include/daggy/ThreadPool.hpp
View File

@@ -24,7 +24,7 @@ namespace daggy {
class TaskQueue { class TaskQueue {
public: public:
template<class F, class... Args> template<class F, class... Args>
decltype(auto) addTask(F&& f, Args&&... args) { decltype(auto) addTask(F &&f, Args &&... args) {
// using return_type = std::invoke_result<F, Args...>::type; // using return_type = std::invoke_result<F, Args...>::type;
using return_type = std::invoke_result_t<F, Args...>; using return_type = std::invoke_result_t<F, Args...>;
@@ -58,7 +58,7 @@ namespace daggy {
} }
private: private:
std::queue< std::packaged_task<void()> > tasks_; std::queue<std::packaged_task<void()> > tasks_;
std::mutex mtx_; std::mutex mtx_;
}; };
@@ -66,10 +66,7 @@ namespace daggy {
public: public:
explicit ThreadPool(size_t nWorkers) explicit ThreadPool(size_t nWorkers)
: :
tqit_(taskQueues_.begin()) tqit_(taskQueues_.begin()), stop_(false), drain_(false) {
, stop_(false)
, drain_(false)
{
resize(nWorkers); resize(nWorkers);
} }
@@ -78,7 +75,7 @@ namespace daggy {
void shutdown() { void shutdown() {
stop_ = true; stop_ = true;
cv_.notify_all(); cv_.notify_all();
for (std::thread& worker : workers_) { for (std::thread &worker : workers_) {
if (worker.joinable()) if (worker.joinable())
worker.join(); worker.join();
} }
@@ -104,15 +101,15 @@ namespace daggy {
workers_.clear(); workers_.clear();
stop_ = false; stop_ = false;
for(size_t i = 0;i< nWorkers;++i) for (size_t i = 0; i < nWorkers; ++i)
workers_.emplace_back( [&] { workers_.emplace_back([&] {
while (true) { while (true) {
std::packaged_task<void()> task; std::packaged_task<void()> task;
{ {
std::unique_lock<std::mutex> lock(mtx_); std::unique_lock<std::mutex> lock(mtx_);
cv_.wait(lock, [&]{ return stop_ || ! taskQueues_.empty(); }); cv_.wait(lock, [&] { return stop_ || !taskQueues_.empty(); });
if(taskQueues_.empty()) { if (taskQueues_.empty()) {
if(stop_) return; if (stop_) return;
continue; continue;
} }
if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin(); if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin();
@@ -130,7 +127,7 @@ namespace daggy {
}; };
template<class F, class... Args> template<class F, class... Args>
decltype(auto) addTask(F&& f, Args&&... args) { decltype(auto) addTask(F &&f, Args &&... args) {
if (drain_) throw std::runtime_error("Unable to add task to draining pool"); if (drain_) throw std::runtime_error("Unable to add task to draining pool");
auto tq = std::make_shared<TaskQueue>(); auto tq = std::make_shared<TaskQueue>();
@@ -153,7 +150,7 @@ namespace daggy {
private: private:
// need to keep track of threads so we can join them // need to keep track of threads so we can join them
std::vector< std::thread > workers_; std::vector<std::thread> workers_;
// the task queue // the task queue
std::list<std::shared_ptr<TaskQueue>> taskQueues_; std::list<std::shared_ptr<TaskQueue>> taskQueues_;
std::list<std::shared_ptr<TaskQueue>>::iterator tqit_; std::list<std::shared_ptr<TaskQueue>>::iterator tqit_;

1
daggy/include/daggy/executors/ForkingExecutor.hpp
View File

@@ -8,6 +8,7 @@ namespace daggy {
class ForkingExecutor : public Executor { class ForkingExecutor : public Executor {
public: public:
ForkingExecutor() = default; ForkingExecutor() = default;
const std::string getName() const override { return "ForkingExecutor"; } const std::string getName() const override { return "ForkingExecutor"; }
AttemptRecord runCommand(std::vector<std::string> cmd) override; AttemptRecord runCommand(std::vector<std::string> cmd) override;

13
daggy/src/DAG.cpp
View File

@@ -3,6 +3,7 @@
namespace daggy { namespace daggy {
size_t DAG::size() const { return vertices_.size(); } size_t DAG::size() const { return vertices_.size(); }
bool DAG::empty() const { return vertices_.empty(); } bool DAG::empty() const { return vertices_.empty(); }
size_t DAG::addVertex() { size_t DAG::addVertex() {
@@ -27,7 +28,7 @@ namespace daggy {
bool DAG::hasPath(const size_t from, const size_t to) const { bool DAG::hasPath(const size_t from, const size_t to) const {
if (from >= vertices_.size()) throw std::runtime_error("No such vertex " + std::to_string(from)); if (from >= vertices_.size()) throw std::runtime_error("No such vertex " + std::to_string(from));
if (to >= vertices_.size()) throw std::runtime_error("No such vertex " + std::to_string(to)); if (to >= vertices_.size()) throw std::runtime_error("No such vertex " + std::to_string(to));
for (const auto & child : vertices_[from].children) { for (const auto &child : vertices_[from].children) {
if (child == to) return true; if (child == to) return true;
if (hasPath(child, to)) return true; if (hasPath(child, to)) return true;
} }
@@ -37,13 +38,13 @@ namespace daggy {
void DAG::reset() { void DAG::reset() {
// Reset the state of all vertices // Reset the state of all vertices
for (auto & v : vertices_) { for (auto &v : vertices_) {
v.state = VertexState::UNVISITED; v.state = VertexState::UNVISITED;
v.depCount = 0; v.depCount = 0;
} }
// Calculate the upstream count // Calculate the upstream count
for (auto & v : vertices_) { for (auto &v : vertices_) {
for (auto c : v.children) { for (auto c : v.children) {
++vertices_[c].depCount; ++vertices_[c].depCount;
} }
@@ -51,7 +52,7 @@ namespace daggy {
} }
bool DAG::allVisited() const { bool DAG::allVisited() const {
for (const auto & v : vertices_) { for (const auto &v : vertices_) {
if (v.state != VertexState::VISITED) return false; if (v.state != VertexState::VISITED) return false;
} }
return true; return true;
@@ -59,7 +60,7 @@ namespace daggy {
std::optional<const size_t> DAG::visitNext() { std::optional<const size_t> DAG::visitNext() {
for (size_t i = 0; i < vertices_.size(); ++i) { for (size_t i = 0; i < vertices_.size(); ++i) {
auto & v = vertices_[i]; auto &v = vertices_[i];
if (v.state != VertexState::UNVISITED) continue; if (v.state != VertexState::UNVISITED) continue;
if (v.depCount != 0) continue; if (v.depCount != 0) continue;
@@ -70,7 +71,7 @@ namespace daggy {
} }
void DAG::completeVisit(const size_t id) { void DAG::completeVisit(const size_t id) {
auto & v = vertices_[id]; auto &v = vertices_[id];
v.state = VertexState::VISITED; v.state = VertexState::VISITED;
for (auto c : v.children) { for (auto c : v.children) {
--vertices_[c].depCount; --vertices_[c].depCount;

58
daggy/src/Scheduler.cpp
View File

@@ -3,13 +3,8 @@
using namespace std::chrono_literals; using namespace std::chrono_literals;
namespace daggy { namespace daggy {
Scheduler::Scheduler(Executor & executor Scheduler::Scheduler(Executor &executor, size_t executorThreads, size_t schedulerThreads)
, size_t executorThreads : executor_(executor), schedulers_(schedulerThreads), executors_(executorThreads) {}
, size_t schedulerThreads)
: executor_(executor)
, schedulers_(schedulerThreads)
, executors_(executorThreads)
{ }
Scheduler::~Scheduler() { Scheduler::~Scheduler() {
@@ -18,25 +13,22 @@ namespace daggy {
} }
std::future<void> std::future<void>
Scheduler::scheduleDAG(std::string runName Scheduler::scheduleDAG(std::string runName, std::vector<Task> tasks,
, std::vector<Task> tasks std::unordered_map<std::string, ParameterValue> parameters, DAG dag
, std::unordered_map<std::string, ParameterValue> parameters ) {
, DAG dag // Initialize the dag if one wasn't provided
)
{
// Initialize the dag
if (dag.empty()) { if (dag.empty()) {
std::unordered_map<std::string, size_t> tids; std::unordered_map<std::string, size_t> taskIDs;
// Add all the vertices // Add all the vertices
for (size_t i = 0; i < tasks.size(); ++i) { for (const auto &task : tasks) {
tids[tasks[i].name] = dag.addVertex(); taskIDs[task.name] = dag.addVertex();
} }
// Add edges // Add edges
for (size_t i = 0; i < tasks.size(); ++i) { for (size_t i = 0; i < tasks.size(); ++i) {
for (const auto & c : tasks[i].children) { for (const auto &c : tasks[i].children) {
dag.addEdge(i, tids[c]); dag.addEdge(i, taskIDs[c]);
} }
} }
dag.reset(); dag.reset();
@@ -44,10 +36,10 @@ namespace daggy {
// Create the DAGRun // Create the DAGRun
std::lock_guard<std::mutex> guard(mtx_); std::lock_guard<std::mutex> guard(mtx_);
auto & dr = runs_[runName]; auto &dr = runs_[runName];
dr.tasks = tasks; dr.tasks = tasks;
dr.parameters = parameters; dr.parameters = std::move(parameters);
dr.dag = dag; dr.dag = dag;
dr.taskRuns = std::vector<TaskRun>{tasks.size()}; dr.taskRuns = std::vector<TaskRun>{tasks.size()};
@@ -55,20 +47,19 @@ namespace daggy {
return std::move(schedulers_.addTask([&]() { runDAG(runName, dr); })); return std::move(schedulers_.addTask([&]() { runDAG(runName, dr); }));
} }
void Scheduler::runDAG(const std::string & name, DAGRun & run) void Scheduler::runDAG(const std::string &name, DAGRun &run) {
{ struct TaskState {
struct Task {
size_t tid; size_t tid;
std::future<std::vector<AttemptRecord>> fut; std::future<std::vector<AttemptRecord>> fut;
bool complete; bool complete;
}; };
std::vector<Task> tasks; std::vector<TaskState> tasks;
while (! run.dag.allVisited()) { while (!run.dag.allVisited()) {
// Check for any completed tasks // Check for any completed tasks
for (auto & task : tasks) { for (auto &task : tasks) {
if (task.complete) continue; if (task.complete) continue;
if (task.fut.valid()) { if (task.fut.valid()) {
@@ -80,13 +71,13 @@ namespace daggy {
} }
} }
// Get the next dag to run // Add all remaining tasks in a task queue to avoid dominating the thread pool
auto tq = std::make_shared<TaskQueue>();
auto t = run.dag.visitNext(); auto t = run.dag.visitNext();
while (t.has_value()) { while (t.has_value()) {
// Schedule the task to run // Schedule the task to run
Task tsk{ .tid = t.value() TaskState tsk{.tid = t.value(), .fut = tq->addTask(
, .fut = executors_.addTask([&](){return runTask(run.tasks[t.value()]);}) [&]() { return runTask(run.tasks[t.value()]); }), .complete = false
, .complete = false
}; };
tasks.push_back(std::move(tsk)); tasks.push_back(std::move(tsk));
@@ -95,13 +86,16 @@ namespace daggy {
if (not nt.has_value()) break; if (not nt.has_value()) break;
t.emplace(nt.value()); t.emplace(nt.value());
} }
if (! tq->empty()) {
executors_.addTasks(tq);
}
std::this_thread::sleep_for(250ms); std::this_thread::sleep_for(250ms);
} }
} }
std::vector<AttemptRecord> std::vector<AttemptRecord>
Scheduler::runTask(const Task & task) { Scheduler::runTask(const Task &task) {
std::vector<AttemptRecord> attempts; std::vector<AttemptRecord> attempts;
while (attempts.size() < task.max_retries) { while (attempts.size() < task.max_retries) {

9
daggy/src/Server.cpp
View File

@@ -5,8 +5,7 @@ using namespace Pistache;
namespace daggy { namespace daggy {
void Server::init(int threads) { void Server::init(int threads) {
auto opts = Http::Endpoint::options() auto opts = Http::Endpoint::options()
.threads(threads) .threads(threads);
;
endpoint_.init(opts); endpoint_.init(opts);
createDescription(); createDescription();
} }
@@ -21,11 +20,11 @@ namespace daggy {
void Server::createDescription() { void Server::createDescription() {
desc_ desc_
.info() .info()
.license("Apache", "http://www.apache.org/licenses/LICENSE-2.0") .license("Apache", "http://www.apache.org/licenses/LICENSE-2.0");
;
auto backendErrorResponse = desc_.response(Http::Code::Internal_Server_Error, "An error occured with the backend"); auto backendErrorResponse = desc_.response(Http::Code::Internal_Server_Error,
"An error occured with the backend");
desc_ desc_
.schemes(Rest::Scheme::Http) .schemes(Rest::Scheme::Http)

17
daggy/src/executors/ForkingExecutor.cpp
View File

@@ -16,7 +16,7 @@ std::string slurp(int fd) {
const ssize_t BUFFER_SIZE = 4096; const ssize_t BUFFER_SIZE = 4096;
char buffer[BUFFER_SIZE]; char buffer[BUFFER_SIZE];
struct pollfd pfd{ .fd = fd, .events = POLLIN, .revents = 0 }; struct pollfd pfd{.fd = fd, .events = POLLIN, .revents = 0};
poll(&pfd, 1, 1); poll(&pfd, 1, 1);
while (pfd.revents & POLLIN) { while (pfd.revents & POLLIN) {
@@ -34,22 +34,23 @@ std::string slurp(int fd) {
} }
daggy::AttemptRecord daggy::AttemptRecord
ForkingExecutor::runCommand(std::vector<std::string> cmd) ForkingExecutor::runCommand(std::vector<std::string> cmd) {
{
AttemptRecord rec; AttemptRecord rec;
rec.startTime = Clock::now(); rec.startTime = Clock::now();
// Need to convert the strings // Need to convert the strings
std::vector<char *> argv; std::vector<char *> argv;
for (const auto & s : cmd) { for (const auto &s : cmd) {
argv.push_back(const_cast<char *>(s.c_str())); argv.push_back(const_cast<char *>(s.c_str()));
} }
argv.push_back(nullptr); argv.push_back(nullptr);
// Create the pipe // Create the pipe
int stdoutPipe[2]; pipe2(stdoutPipe, O_DIRECT); int stdoutPipe[2];
int stderrPipe[2]; pipe2(stderrPipe, O_DIRECT); pipe2(stdoutPipe, O_DIRECT);
int stderrPipe[2];
pipe2(stderrPipe, O_DIRECT);
pid_t child = fork(); pid_t child = fork();
if (child < 0) { if (child < 0) {
@@ -64,8 +65,8 @@ daggy::AttemptRecord
} }
std::atomic<bool> running = true; std::atomic<bool> running = true;
std::thread stdoutReader([&]() { while(running) rec.output.append(slurp(stdoutPipe[0])); }); std::thread stdoutReader([&]() { while (running) rec.output.append(slurp(stdoutPipe[0])); });
std::thread stderrReader([&]() { while(running) rec.error.append(slurp(stderrPipe[0])); }); std::thread stderrReader([&]() { while (running) rec.error.append(slurp(stderrPipe[0])); });
int rc = 0; int rc = 0;
waitpid(child, &rc, 0); waitpid(child, &rc, 0);

6
tests/unit_scheduler.cpp
View File

@@ -17,7 +17,9 @@ TEST_CASE("Basic Scheduler Execution", "[scheduler]") {
}; };
SECTION("Simple Run") { SECTION("Simple Run") {
auto fut = sched.scheduleDAG("Simple", tasks, {}); auto fut_a = sched.scheduleDAG("Simple 1", tasks, {});
fut.get(); auto fut_b = sched.scheduleDAG("Simple 2", tasks, {});
fut_a.get();
fut_b.get();
} }
} }