Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Scheduler now completing work

Browse Source
This commit is contained in:
Ian Roddis
2021-07-06 11:35:30 -03:00
parent 9b9409d504
commit a603ebead5
5 changed files with 45 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
daggy/include/daggy/Scheduler.hpp
View File

@@ -32,7 +32,8 @@ namespace daggy {
~Scheduler(); ~Scheduler();
// returns DagRun ID // returns DagRun ID
void scheduleDAG(std::string runName std::future<void>
scheduleDAG(std::string runName
, std::vector<Task> tasks , std::vector<Task> tasks
, std::unordered_map<std::string, ParameterValue> parameters , std::unordered_map<std::string, ParameterValue> parameters
, DAG dag = {} // Allows for loading of an existing DAG , DAG dag = {} // Allows for loading of an existing DAG
@@ -61,6 +62,7 @@ namespace daggy {
std::vector<AttemptRecord> runTask(const Task & task); std::vector<AttemptRecord> runTask(const Task & task);
std::unordered_map<std::string, DAGRun> runs_; std::unordered_map<std::string, DAGRun> runs_;
std::vector<std::future<void>> futs_;
Executor & executor_; Executor & executor_;
ThreadPool schedulers_; ThreadPool schedulers_;
ThreadPool executors_; ThreadPool executors_;

41
daggy/include/daggy/ThreadPool.hpp
View File

@@ -10,6 +10,8 @@
#include <functional> #include <functional>
#include <list> #include <list>
using namespace std::chrono_literals;
namespace daggy { namespace daggy {
/* /*
@@ -66,6 +68,7 @@ namespace daggy {
: :
tqit_(taskQueues_.begin()) tqit_(taskQueues_.begin())
, stop_(false) , stop_(false)
, drain_(false)
{ {
resize(nWorkers); resize(nWorkers);
} }
@@ -82,35 +85,45 @@ namespace daggy {
} }
void drain() { void drain() {
resize(workers_.size()); drain_ = true;
while (true) {
{
std::lock_guard<std::mutex> guard(mtx_);
if (taskQueues_.empty()) break;
}
std::this_thread::sleep_for(250ms);
}
}
void restart() {
drain_ = false;
} }
void resize(size_t nWorkers) { void resize(size_t nWorkers) {
shutdown(); shutdown();
workers_.clear(); workers_.clear();
stop_ = false;
for(size_t i = 0;i< nWorkers;++i) for(size_t i = 0;i< nWorkers;++i)
workers_.emplace_back( [&] { workers_.emplace_back( [&] {
for(;;) { while (true) {
std::packaged_task<void()> task; std::packaged_task<void()> task;
{ {
std::unique_lock<std::mutex> lock(mtx_); std::unique_lock<std::mutex> lock(mtx_);
cv_.wait(lock, [&]{ return stop_ || !taskQueues_.empty(); }); cv_.wait(lock, [&]{ return stop_ || ! taskQueues_.empty(); });
if(taskQueues_.empty()) { if(taskQueues_.empty()) {
if(stop_) return; if(stop_) return;
continue; continue;
} }
if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin(); if (tqit_ == taskQueues_.end()) tqit_ = taskQueues_.begin();
if (not (*tqit_)->empty()) { task = std::move((*tqit_)->pop());
task = std::move((*tqit_)->pop()); if ((*tqit_)->empty()) {
if ((*tqit_)->empty()) { tqit_ = taskQueues_.erase(tqit_);
tqit_ = taskQueues_.erase(tqit_); } else {
} else { tqit_++;
tqit_++;
}
} }
} }
task(); task();
} }
} }
); );
@@ -118,6 +131,7 @@ namespace daggy {
template<class F, class... Args> template<class F, class... Args>
decltype(auto) addTask(F&& f, Args&&... args) { decltype(auto) addTask(F&& f, Args&&... args) {
if (drain_) throw std::runtime_error("Unable to add task to draining pool");
auto tq = std::make_shared<TaskQueue>(); auto tq = std::make_shared<TaskQueue>();
auto fut = tq->addTask(f, args...); auto fut = tq->addTask(f, args...);
@@ -130,7 +144,8 @@ namespace daggy {
return fut; return fut;
} }
void addTaskQueue(std::shared_ptr<TaskQueue> tq) { void addTasks(std::shared_ptr<TaskQueue> tq) {
if (drain_) throw std::runtime_error("Unable to add task to draining pool");
std::lock_guard<std::mutex> guard(mtx_); std::lock_guard<std::mutex> guard(mtx_);
taskQueues_.push_back(tq); taskQueues_.push_back(tq);
cv_.notify_one(); cv_.notify_one();
@@ -140,7 +155,6 @@ namespace daggy {
// need to keep track of threads so we can join them // need to keep track of threads so we can join them
std::vector< std::thread > workers_; std::vector< std::thread > workers_;
// the task queue // the task queue
std::queue< std::packaged_task<void()> > tasks_;
std::list<std::shared_ptr<TaskQueue>> taskQueues_; std::list<std::shared_ptr<TaskQueue>> taskQueues_;
std::list<std::shared_ptr<TaskQueue>>::iterator tqit_; std::list<std::shared_ptr<TaskQueue>>::iterator tqit_;
@@ -148,6 +162,7 @@ namespace daggy {
std::mutex mtx_; std::mutex mtx_;
std::condition_variable cv_; std::condition_variable cv_;
std::atomic<bool> stop_; std::atomic<bool> stop_;
std::atomic<bool> drain_;
}; };
} }

29
daggy/src/Scheduler.cpp
View File

@@ -17,7 +17,8 @@ namespace daggy {
schedulers_.shutdown(); schedulers_.shutdown();
} }
void Scheduler::scheduleDAG(std::string runName std::future<void>
Scheduler::scheduleDAG(std::string runName
, std::vector<Task> tasks , std::vector<Task> tasks
, std::unordered_map<std::string, ParameterValue> parameters , std::unordered_map<std::string, ParameterValue> parameters
, DAG dag , DAG dag
@@ -42,17 +43,16 @@ namespace daggy {
} }
// Create the DAGRun // Create the DAGRun
{ std::lock_guard<std::mutex> guard(mtx_);
std::lock_guard<std::mutex> guard(mtx_); auto & dr = runs_[runName];
auto & dr = runs_[runName];
dr.tasks = tasks; dr.tasks = tasks;
dr.parameters = parameters; dr.parameters = parameters;
dr.dag = dag; dr.dag = dag;
dr.taskRuns = std::vector<TaskRun>{tasks.size()}; dr.taskRuns = std::vector<TaskRun>{tasks.size()};
schedulers_.addTask([&]() { runDAG(runName, dr); }); // return std::move(schedulers_.addTask([&]() { runDAG(runName, dr); }));
} return std::move(schedulers_.addTask([&]() { runDAG(runName, dr); }));
} }
void Scheduler::runDAG(const std::string & name, DAGRun & run) void Scheduler::runDAG(const std::string & name, DAGRun & run)
@@ -65,20 +65,15 @@ namespace daggy {
std::vector<Task> tasks; std::vector<Task> tasks;
std::cout << "Running dag " << name << std::endl;
while (! run.dag.allVisited()) { while (! run.dag.allVisited()) {
// Check for any completed tasks // Check for any completed tasks
std::cout << "Polling completed" << std::endl;
for (auto & task : tasks) { for (auto & task : tasks) {
if (task.complete) continue; if (task.complete) continue;
if (task.fut.valid()) { if (task.fut.valid()) {
std::cout << "Checking tid " << task.tid << std::endl;
auto ars = task.fut.get(); auto ars = task.fut.get();
std::cout << "Got" << std::endl;
if (ars.back().rc == 0) { if (ars.back().rc == 0) {
std::cout << "Completing node " << task.tid << std::endl;
run.dag.completeVisit(task.tid); run.dag.completeVisit(task.tid);
} }
task.complete = true; task.complete = true;
@@ -86,10 +81,8 @@ namespace daggy {
} }
// Get the next dag to run // Get the next dag to run
std::cout << "Polling scheduling" << std::endl;
auto t = run.dag.visitNext(); auto t = run.dag.visitNext();
while (t.has_value()) { while (t.has_value()) {
std::cout << "Scheduling " << t.value() << std::endl;
// Schedule the task to run // Schedule the task to run
Task tsk{ .tid = t.value() Task tsk{ .tid = t.value()
, .fut = executors_.addTask([&](){return runTask(run.tasks[t.value()]);}) , .fut = executors_.addTask([&](){return runTask(run.tasks[t.value()]);})
@@ -103,7 +96,6 @@ namespace daggy {
t.emplace(nt.value()); t.emplace(nt.value());
} }
std::cout << "sleeping" << std::endl;
std::this_thread::sleep_for(250ms); std::this_thread::sleep_for(250ms);
} }
} }
@@ -113,7 +105,6 @@ namespace daggy {
std::vector<AttemptRecord> attempts; std::vector<AttemptRecord> attempts;
while (attempts.size() < task.max_retries) { while (attempts.size() < task.max_retries) {
std::cout << "Attempt " << attempts.size() << ": Running " << task.command.front() << std::endl;
attempts.push_back(executor_.runCommand(task.command)); attempts.push_back(executor_.runCommand(task.command));
if (attempts.back().rc == 0) break; if (attempts.back().rc == 0) break;
} }

4
tests/unit_scheduler.cpp
View File

@@ -17,7 +17,7 @@ TEST_CASE("Basic Scheduler Execution", "[scheduler]") {
}; };
SECTION("Simple Run") { SECTION("Simple Run") {
sched.scheduleDAG("Simple", tasks, {}); auto fut = sched.scheduleDAG("Simple", tasks, {});
sched.drain(); fut.get();
} }
} }

4
tests/unit_threadpool.cpp
View File

@@ -13,12 +13,12 @@ TEST_CASE("Threadpool Construction", "[threadpool]") {
std::vector<std::future<uint32_t>> rets; std::vector<std::future<uint32_t>> rets;
SECTION("Simple runs") { SECTION("Adding large tasks queues with return values") {
auto tq = std::make_shared<daggy::TaskQueue>(); auto tq = std::make_shared<daggy::TaskQueue>();
std::vector<std::future<uint32_t>> res; std::vector<std::future<uint32_t>> res;
for (size_t i = 0; i < 100; ++i) for (size_t i = 0; i < 100; ++i)
res.emplace_back(std::move(tq->addTask([&cnt]() { cnt++; return cnt.load(); }))); res.emplace_back(std::move(tq->addTask([&cnt]() { cnt++; return cnt.load(); })));
tp.addTaskQueue(tq); tp.addTasks(tq);
for (auto & r : res) r.get(); for (auto & r : res) r.get();
REQUIRE(cnt == 100); REQUIRE(cnt == 100);
} }