Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding support for execution on slurm grids

Browse Source 
- Adding support for SlurmTaskExecutor in `daggyd` if DAGGY_ENABLE_SLURM is defined.
- Renaming some test cases
- Enabling compile-time slurm support
- Adding slurm documentation
This commit is contained in:
Ian Roddis
2021-09-10 10:53:58 -03:00
parent d15580f47f
commit d731f9f5b1
19 changed files with 460 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/unit_dag.cpp
View File

@@ -4,7 +4,7 @@
#include <catch2/catch.hpp>
TEST_CASE("DAG Construction Tests", "[dag]") {
TEST_CASE("dag_construction", "[dag]") {
daggy::DAG<size_t, size_t> dag;
REQUIRE(dag.size() == 0);
@@ -34,7 +34,7 @@ TEST_CASE("DAG Construction Tests", "[dag]") {
}
}
TEST_CASE("DAG Traversal Tests", "[dag]") {
TEST_CASE("dag_traversal", "[dag]") {
daggy::DAG<size_t, size_t> dag;
const int N_VERTICES = 10;

2
tests/unit_dagrun_loggers.cpp
View File

@@ -54,7 +54,7 @@ TEST_CASE("Filesystem Logger", "[filesystem_logger]") {
}
*/
TEST_CASE("ostream Logger", "[ostream_logger]") {
TEST_CASE("ostream_logger", "[ostream_logger]") {
//cleanup();
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);

8
tests/unit_executor_forkingexecutor.cpp
View File

@@ -7,7 +7,7 @@
#include <catch2/catch.hpp>
TEST_CASE("Basic Execution", "[forking_executor]") {
TEST_CASE("forking_executor", "[forking_executor]") {
daggy::executors::task::ForkingTaskExecutor ex(10);
SECTION("Simple Run") {
@@ -16,7 +16,7 @@ TEST_CASE("Basic Execution", "[forking_executor]") {
REQUIRE(ex.validateTaskParameters(task.job));
auto rec = ex.execute(task);
auto rec = ex.execute("command", task);
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() >= 6);
@@ -27,7 +27,7 @@ TEST_CASE("Basic Execution", "[forking_executor]") {
daggy::Task task{.job{
{"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}};
auto rec = ex.execute(task);
auto rec = ex.execute("command", task);
REQUIRE(rec.rc == 2);
REQUIRE(rec.errorLog.size() >= 20);
@@ -45,7 +45,7 @@ TEST_CASE("Basic Execution", "[forking_executor]") {
daggy::Task task{.job{
{"command", daggy::executors::task::ForkingTaskExecutor::Command{"/usr/bin/cat", bigFile}}}};
auto rec = ex.execute(task);
auto rec = ex.execute("command", task);
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));

105
tests/unit_executor_slurmexecutor.cpp Normal file
View File

@@ -0,0 +1,105 @@
#include <iostream>
#include <filesystem>
#include "daggy/executors/task/SlurmTaskExecutor.hpp"
#include "daggy/Serialization.hpp"
#include "daggy/Utilities.hpp"
#include <catch2/catch.hpp>
namespace fs = std::filesystem;
#ifdef DAGGY_ENABLE_SLURM
TEST_CASE("slurm_execution", "[slurm_executor]") {
daggy::executors::task::SlurmTaskExecutor ex(10);
daggy::ConfigValues defaultJobValues{
{"minCPUs", "1"},
{"minMemoryMB", "100"},
{"minTmpDiskMB", "10"},
{"priority", "1"},
{"timeLimitSeconds", "200"},
{"userID", "1002"},
{"workDir", fs::current_path().string()},
{"tmpDir", fs::current_path().string()}
};
SECTION("Simple Run") {
daggy::Task task{.job{
{"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}}
}};
task.job.merge(defaultJobValues);
REQUIRE(ex.validateTaskParameters(task.job));
auto rec = ex.execute("command", task);
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() >= 6);
REQUIRE(rec.errorLog.empty());
}
SECTION("Error Run") {
daggy::Task task{.job{
{"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/expr", "1", "+", "+"}}}};
task.job.merge(defaultJobValues);
auto rec = ex.execute("command", task);
REQUIRE(rec.rc != 0);
REQUIRE(rec.errorLog.size() >= 20);
REQUIRE(rec.outputLog.empty());
}
SECTION("Large Output") {
const std::vector<std::string> BIG_FILES{
"/usr/share/dict/linux.words", "/usr/share/dict/cracklib-small", "/etc/ssh/moduli"
};
for (const auto &bigFile: BIG_FILES) {
if (!std::filesystem::exists(bigFile)) continue;
daggy::Task task{.job{
{"command", daggy::executors::task::SlurmTaskExecutor::Command{"/usr/bin/cat", bigFile}}}};
task.job.merge(defaultJobValues);
auto rec = ex.execute("command", task);
REQUIRE(rec.rc == 0);
REQUIRE(rec.outputLog.size() == std::filesystem::file_size(bigFile));
REQUIRE(rec.errorLog.empty());
break;
}
}
SECTION("Parameter Expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ]})"};
auto params = daggy::configFromJSON(testParams);
std::string taskJSON = R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
auto result = daggy::expandTaskSet(tasks, ex, params);
REQUIRE(result.size() == 2);
}
SECTION("Build with expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["B"]}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
REQUIRE(tasks.size() == 4);
}
SECTION("Build with expansion using parents instead of children") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}}, "B": {"job": {"command": ["/bin/echo", "B", "{{SOURCE}}", "{{DATE}}"]}, "parents": ["A"]}, "C": {"job": {"command": ["/bin/echo", "C"]}, "parents": ["A"]}})";
auto tasks = daggy::expandTaskSet(daggy::tasksFromJSON(testTasks, defaultJobValues), ex, params);
REQUIRE(tasks.size() == 4);
}
}
#endif

6
tests/unit_serialization.cpp
View File

@@ -8,7 +8,7 @@
namespace fs = std::filesystem;
TEST_CASE("Deserialize Parameters", "[deserialize_parameters]") {
TEST_CASE("parameter_deserialization", "[deserialize_parameters]") {
SECTION("Basic Parse") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name"})"};
auto params = daggy::configFromJSON(testParams);
@@ -27,7 +27,7 @@ TEST_CASE("Deserialize Parameters", "[deserialize_parameters]") {
}
}
TEST_CASE("Task Deserialization", "[deserialize_task]") {
TEST_CASE("task_deserialization", "[deserialize_task]") {
SECTION("Build with no expansion") {
std::string testTasks = R"({
"A": {
@@ -70,7 +70,7 @@ TEST_CASE("Task Deserialization", "[deserialize_task]") {
}
}
TEST_CASE("Task Serialization", "[serialize_tasks]") {
TEST_CASE("task_serialization", "[serialize_tasks]") {
SECTION("Build with no expansion") {
std::string testTasks = R"({"A": {"job": {"command": ["/bin/echo", "A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/echo", "B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/echo", "C"]}}})";
auto tasks = daggy::tasksFromJSON(testTasks);

2
tests/unit_server.cpp
View File

@@ -50,7 +50,7 @@ REQUEST(std::string url, std::string payload = "") {
return response;
}
TEST_CASE("Server Basic Endpoints", "[server_basic]") {
TEST_CASE("rest_endpoint", "[server_basic]") {
std::stringstream ss;
daggy::executors::task::ForkingTaskExecutor executor(10);
daggy::loggers::dag_run::OStreamLogger logger(ss);

6
tests/unit_threadpool.cpp
View File

@@ -7,7 +7,7 @@
using namespace daggy;
TEST_CASE("Threadpool Construction", "[threadpool]") {
TEST_CASE("threadpool", "[threadpool]") {
std::atomic<uint32_t> cnt(0);
ThreadPool tp(10);
@@ -22,7 +22,7 @@ TEST_CASE("Threadpool Construction", "[threadpool]") {
return cnt.load();
})));
tp.addTasks(tq);
for (auto &r : res) r.get();
for (auto &r: res) r.get();
REQUIRE(cnt == 100);
}
@@ -35,7 +35,7 @@ TEST_CASE("Threadpool Construction", "[threadpool]") {
cnt++;
return;
}));
for (auto &r : res) r.get();
for (auto &r: res) r.get();
REQUIRE(cnt == 100);
}
}

6
tests/unit_utilities.cpp
View File

@@ -15,13 +15,13 @@
namespace fs = std::filesystem;
TEST_CASE("String Utilities", "[utilities_string]") {
TEST_CASE("string_utilities", "[utilities_string]") {
std::string test = "/this/is/{{A}}/test/{{A}}";
auto res = daggy::globalSub(test, "{{A}}", "hello");
REQUIRE(res == "/this/is/hello/test/hello");
}
TEST_CASE("Parameter Expansion", "[utilities_parameter_expansion]") {
TEST_CASE("string_expansion", "[utilities_parameter_expansion]") {
SECTION("Basic expansion") {
std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
@@ -55,7 +55,7 @@ TEST_CASE("Parameter Expansion", "[utilities_parameter_expansion]") {
}
}
TEST_CASE("DAG Runner", "[utilities_dag_runner]") {
TEST_CASE("dag_runner", "[utilities_dag_runner]") {
daggy::executors::task::ForkingTaskExecutor ex(10);
std::stringstream ss;
daggy::loggers::dag_run::OStreamLogger logger(ss);