Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small fixes to test cases

Browse Source
This commit is contained in:
Ian Roddis
2022-01-28 10:23:21 -04:00
parent c09d5b7d8b
commit c9820c19a0
11 changed files with 73 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
README.md
View File

@@ -91,7 +91,7 @@ Below is an example DAG Run submission:
"task_one": { "task_one": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/touch", "/bin/touch",
"/tmp/somefile" "/tmp/somefile"
] ]
}, },
@@ -101,7 +101,7 @@ Below is an example DAG Run submission:
"task_two": { "task_two": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/touch", "/bin/touch",
"/tmp/someotherfile" "/tmp/someotherfile"
] ]
}, },
@@ -133,7 +133,7 @@ For instance:
"task_one": { "task_one": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/touch", "/bin/touch",
"{{DIRECTORY}}/{{FILE}}" "{{DIRECTORY}}/{{FILE}}"
] ]
}, },
@@ -167,7 +167,7 @@ Example:
"populate_inputs": { "populate_inputs": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/touch", "/bin/touch",
"{{DIRECTORY}}/{{FILE}}" "{{DIRECTORY}}/{{FILE}}"
] ]
} }
@@ -346,8 +346,8 @@ and capture output.
| Field | Sample | Description | | Field | Sample | Description |
|---------|--------|--------------| |---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run | | command | `[ "/bin/echo", "param1" ]` | The command to run |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. | | commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script | | environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
Slurm Executor (SlurmTaskExecutor) Slurm Executor (SlurmTaskExecutor)
@@ -361,8 +361,8 @@ Required `job` config values:
| Field | Sample | Description | | Field | Sample | Description |
|---------|--------|--------------| |---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run on a slurm host | | command | `[ "/bin/echo", "param1" ]` | The command to run on a slurm host |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. | | commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script | | environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
| minCPUs | `"1"` | Minimum number of CPUs required | | minCPUs | `"1"` | Minimum number of CPUs required |
| minMemoryMB | `"1"` | Minimum memory required, in MB | | minMemoryMB | `"1"` | Minimum memory required, in MB |
@@ -394,8 +394,8 @@ Enforcement via cgroups is planned.
| Field | Sample | Description | | Field | Sample | Description |
|---------|--------|--------------| |---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run | | command | `[ "/bin/echo", "param1" ]` | The command to run |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. | | commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script | | environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
| cores | "1" | Number of cores required by the task | | cores | "1" | Number of cores required by the task |
| memoryMB | "100" | Amount of memory (RSS) required by the task, in MB | | memoryMB | "100" | Amount of memory (RSS) required by the task, in MB |

10
daggyd/tests/unit_server.cpp
View File

@@ -51,8 +51,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
"tag": "unit_server", "tag": "unit_server",
"parameters": { "FILE": [ "A", "B" ] }, "parameters": { "FILE": [ "A", "B" ] },
"tasks": { "tasks": {
"touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ], "environment": []} }, "touch": { "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ], "environment": []} },
"cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ], "environment": []}, "cat": { "job": { "command": [ "/bin/cat", "dagrun_A", "dagrun_B" ], "environment": []},
"parents": [ "touch" ] "parents": [ "touch" ]
} }
} }
@@ -182,9 +182,9 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
std::string dagRunJSON = R"({ std::string dagRunJSON = R"({
"tag": "unit_server", "tag": "unit_server",
"tasks": { "tasks": {
"touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] }, "touch_A": { "job": { "command": [ "/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] }, "sleep_B": { "job": { "command": [ "/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} } "touch_C": { "job": { "command": [ "/bin/touch", "resume_touch_c" ]} }
} }
})"; })";

8
daggyr/tests/unit_server.cpp
View File

@@ -39,7 +39,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task Missing Cores should Fail") SECTION("Task Missing Cores should Fail")
{ {
std::string taskSpec = std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]}, "memoryMB": 100 })"; R"({ "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]}, "memoryMB": 100 })";
auto response = auto response =
HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST"); HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST");
@@ -49,7 +49,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task Missing MemoryMB should Fail") SECTION("Task Missing MemoryMB should Fail")
{ {
std::string taskSpec = std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]}, "cores": 100 })"; R"({ "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]}, "cores": 100 })";
auto response = auto response =
HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST"); HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST");
@@ -59,7 +59,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task submission and get result") SECTION("Task submission and get result")
{ {
std::string taskSpec = std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/echo", "hello", "world" ], "cores": "1", "memoryMB": "100" } })"; R"({ "job": { "command": [ "/bin/echo", "hello", "world" ], "cores": "1", "memoryMB": "100" } })";
// Submit // Submit
{ {
@@ -93,7 +93,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task capacity changes") SECTION("Task capacity changes")
{ {
std::string taskSpec = std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/sleep", "5" ], "cores": "1", "memoryMB": "100" } })"; R"({ "job": { "command": [ "/bin/sleep", "5" ], "cores": "1", "memoryMB": "100" } })";
auto getCapacity = [&]() -> daggy::executors::task::daggy_runner::Capacity { auto getCapacity = [&]() -> daggy::executors::task::daggy_runner::Capacity {
daggy::executors::task::daggy_runner::Capacity cap; daggy::executors::task::daggy_runner::Capacity cap;

12
examples/sample_dag.json
View File

@@ -10,7 +10,7 @@
"A": { "A": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/echo", "/bin/echo",
"A", "A",
"{{DATE}}" "{{DATE}}"
] ]
@@ -24,7 +24,7 @@
"b": { "b": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/sleep", "/bin/sleep",
"2" "2"
] ]
}, },
@@ -35,7 +35,7 @@
"c": { "c": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/echo", "/bin/echo",
"C", "C",
"{{DATE}}" "{{DATE}}"
] ]
@@ -48,7 +48,7 @@
"d": { "d": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/echo", "/bin/echo",
"D", "D",
"{{DATE}}" "{{DATE}}"
] ]
@@ -60,7 +60,7 @@
"e": { "e": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/echo", "/bin/echo",
"E", "E",
"{{DATE}}" "{{DATE}}"
] ]
@@ -72,7 +72,7 @@
"f": { "f": {
"job": { "job": {
"command": [ "command": [
"/usr/bin/echo", "/bin/echo",
"F", "F",
"{{DATE}}" "{{DATE}}"
] ]

28
libdaggy/tests/unit_dagrunner.cpp
View File

@@ -24,11 +24,11 @@ TEST_CASE("dagrunner", "[dagrunner][dagrunner_order_preservation]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams); dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string taskJSON = R"({ std::string taskJSON = R"({
"A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]}, "A": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]}, "B": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]}, "C": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
"D": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "E"]}, "D": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "E"]},
"E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}} "E": {"job": {"command": ["/bin/touch", "{{DATE}}"]}}
})"; })";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex, dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex,
@@ -83,10 +83,10 @@ TEST_CASE("DAGRunner simple execution", "[dagrunner][dagrunner_simple]")
std::unordered_map<std::string, std::string> files{ std::unordered_map<std::string, std::string> files{
{"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}}; {"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}};
std::string taskJSON = std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + files.at("A") + R"({"A": {"job": {"command": ["/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" + R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
files.at("B") + files.at("B") +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" + R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
files.at("C") + R"("]}}})"; files.at("C") + R"("]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks); auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -132,10 +132,10 @@ TEST_CASE("DAG Runner Restart old DAG", "[dagrunner][dagrunner_restart]")
std::string goodPrefix = "rec_error_"; std::string goodPrefix = "rec_error_";
std::string badPrefix = "noexist/rec_error_"; std::string badPrefix = "noexist/rec_error_";
std::string taskJSON = std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + goodPrefix + R"({"A": {"job": {"command": ["/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" + R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
badPrefix + badPrefix +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" + R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
badPrefix + R"(C"]}}})"; badPrefix + R"(C"]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex); dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks); auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -179,7 +179,7 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams); dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string generatorOutput = std::string generatorOutput =
R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})"; R"({"B": {"job": {"command": ["/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
fs::path ofn = fs::current_path() / "generator_test_output.json"; fs::path ofn = fs::current_path() / "generator_test_output.json";
std::ofstream ofh{ofn}; std::ofstream ofh{ofn};
ofh << generatorOutput << std::endl; ofh << generatorOutput << std::endl;
@@ -188,10 +188,10 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
daggy::TimePoint globalStartTime = daggy::Clock::now(); daggy::TimePoint globalStartTime = daggy::Clock::now();
std::stringstream jsonTasks; std::stringstream jsonTasks;
jsonTasks jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", )" << R"({ "A": { "job": {"command": [ "/bin/cat", )"
<< std::quoted(ofn.string()) << std::quoted(ofn.string())
<< R"(]}, "children": ["C"], "isGenerator": true},)" << R"(]}, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })"; << R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str()); dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2); REQUIRE(dagSpec.tasks.size() == 2);

14
libdaggy/tests/unit_executor_forkingexecutor.cpp
View File

@@ -18,7 +18,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{ {
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}}; "/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job)); REQUIRE(ex.validateTaskParameters(task.job));
@@ -32,7 +32,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Simple Run using commandString") SECTION("Simple Run using commandString")
{ {
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}}; daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
REQUIRE(ex.validateTaskParameters(task.job)); REQUIRE(ex.validateTaskParameters(task.job));
@@ -100,7 +100,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{ {
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}}; "/bin/sleep", "30"}}}};
auto start = daggy::Clock::now(); auto start = daggy::Clock::now();
auto recFuture = ex.execute(0, "command", task); auto recFuture = ex.execute(0, "command", task);
@@ -120,8 +120,8 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Large Output") SECTION("Large Output")
{ {
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words", const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/usr/share/dict/cracklib-small", "/share/dict/cracklib-small",
"/etc/ssh/moduli"}; "/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) { for (const auto &bigFile : BIG_FILES) {
@@ -130,7 +130,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{ .job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}}; "/bin/cat", bigFile}}}};
auto recFuture = ex.execute(0, "command", task); auto recFuture = ex.execute(0, "command", task);
auto rec = recFuture->get(); auto rec = recFuture->get();
@@ -147,7 +147,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::string taskJSON = std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON); auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params); auto result = daggy::expandTaskSet(tasks, ex, params);

4
libdaggy/tests/unit_executor_noopexecutor.cpp
View File

@@ -18,7 +18,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
{ {
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::NoopTaskExecutor::Command{ .job{{"command", daggy::executors::task::NoopTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}}; "/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job)); REQUIRE(ex.validateTaskParameters(task.job));
@@ -34,7 +34,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::string taskJSON = std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})"; R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON); auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params); auto result = daggy::expandTaskSet(tasks, ex, params);

16
libdaggy/tests/unit_executor_slurmexecutor.cpp
View File

@@ -43,7 +43,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run") SECTION("Simple Run")
{ {
daggy::Task task{.job{ daggy::Task task{.job{
{"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}}}}; {"command", std::vector<std::string>{"/bin/echo", "abc", "123"}}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
@@ -114,7 +114,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run using commandString") SECTION("Simple Run using commandString")
{ {
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}}; daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
REQUIRE(ex.validateTaskParameters(task.job)); REQUIRE(ex.validateTaskParameters(task.job));
@@ -131,7 +131,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{ {
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{ .job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/expr", "1", "+", "+"}}}}; "/bin/expr", "1", "+", "+"}}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task); auto recFuture = ex.execute(0, "command", task);
@@ -146,7 +146,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{ {
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{ .job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}}; "/bin/sleep", "30"}}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task); auto recFuture = ex.execute(0, "command", task);
@@ -161,8 +161,8 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Large Output") SECTION("Large Output")
{ {
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words", const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/usr/share/dict/cracklib-small", "/share/dict/cracklib-small",
"/etc/ssh/moduli"}; "/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) { for (const auto &bigFile : BIG_FILES) {
@@ -171,7 +171,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
daggy::Task task{ daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{ .job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}}; "/bin/cat", bigFile}}}};
task.job.merge(defaultJobValues); task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task); auto recFuture = ex.execute(0, "command", task);
@@ -190,7 +190,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::string taskJSON = std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})"; R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues); auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
auto result = daggy::expandTaskSet(tasks, ex, params); auto result = daggy::expandTaskSet(tasks, ex, params);

10
libdaggy/tests/unit_server.cpp
View File

@@ -163,8 +163,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
"tag": "unit_server", "tag": "unit_server",
"parameters": { "FILE": [ "A", "B" ] }, "parameters": { "FILE": [ "A", "B" ] },
"tasks": { "tasks": {
"touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]} }, "touch": { "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]} },
"cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ]}, "cat": { "job": { "command": [ "/bin/cat", "dagrun_A", "dagrun_B" ]},
"parents": [ "touch" ] "parents": [ "touch" ]
} }
} }
@@ -293,9 +293,9 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
std::string dagRunJSON = R"({ std::string dagRunJSON = R"({
"tag": "unit_server", "tag": "unit_server",
"tasks": { "tasks": {
"touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] }, "touch_A": { "job": { "command": [ "/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] }, "sleep_B": { "job": { "command": [ "/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} } "touch_C": { "job": { "command": [ "/bin/touch", "resume_touch_c" ]} }
} }
})"; })";

7
libdaggy/tests/unit_threadpool.cpp
View File

@@ -4,6 +4,7 @@
#include "daggy/ThreadPool.hpp" #include "daggy/ThreadPool.hpp"
using namespace std::chrono_literals;
using namespace daggy; using namespace daggy;
TEST_CASE("threadpool", "[threadpool]") TEST_CASE("threadpool", "[threadpool]")
@@ -36,8 +37,12 @@ TEST_CASE("threadpool", "[threadpool]")
cnt++; cnt++;
return; return;
})); }));
for (auto &r : res) for (auto &r : res) {
while (! r->ready()) {
std::this_thread::sleep_for(150ms);
}
r->get(); r->get();
}
REQUIRE(cnt == 100); REQUIRE(cnt == 100);
} }
} }

12
libdaggy/tests/unit_utilities.cpp
View File

@@ -41,7 +41,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{ std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}", std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}",
"{{TYPE}}"}; "{{TYPE}}"};
auto allCommands = daggy::interpolateValues(cmd, params); auto allCommands = daggy::interpolateValues(cmd, params);
@@ -53,7 +53,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{ std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo"}; std::vector<std::string> cmd{"/bin/echo"};
auto allCommands = daggy::interpolateValues(cmd, params); auto allCommands = daggy::interpolateValues(cmd, params);
REQUIRE(allCommands.size() == 1); REQUIRE(allCommands.size() == 1);
@@ -64,7 +64,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{ std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"}; std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}"};
auto allCommands = daggy::interpolateValues(cmd, params); auto allCommands = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE| // TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -76,7 +76,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{ std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"}; R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams); auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"}; std::vector<std::string> cmd{"/bin/touch", "{{DATE}}_{{SOURCE}}"};
auto result = daggy::interpolateValues(cmd, params); auto result = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE| // TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -93,9 +93,9 @@ TEST_CASE("expand_taskset", "[utilities][expand_taskset]")
std::stringstream jsonTasks; std::stringstream jsonTasks;
jsonTasks jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", "/etc/passwd"]} )" << R"({ "A": { "job": {"command": [ "/bin/cat", "/etc/passwd"]} )"
<< R"(, "children": ["C"], "isGenerator": true},)" << R"(, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })"; << R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str()); dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2); REQUIRE(dagSpec.tasks.size() == 2);