Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small fixes to test cases

Browse Source
This commit is contained in:
Ian Roddis
2022-01-28 10:23:21 -04:00
parent c09d5b7d8b
commit c9820c19a0
11 changed files with 73 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
libdaggy/tests/unit_dagrunner.cpp
View File

@@ -24,11 +24,11 @@ TEST_CASE("dagrunner", "[dagrunner][dagrunner_order_preservation]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string taskJSON = R"({
"A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
"D": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "E"]},
"E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}}
"A": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
"D": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "E"]},
"E": {"job": {"command": ["/bin/touch", "{{DATE}}"]}}
})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex,
@@ -83,10 +83,10 @@ TEST_CASE("DAGRunner simple execution", "[dagrunner][dagrunner_simple]")
std::unordered_map<std::string, std::string> files{
{"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}};
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
R"({"A": {"job": {"command": ["/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
files.at("B") +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
files.at("C") + R"("]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -132,10 +132,10 @@ TEST_CASE("DAG Runner Restart old DAG", "[dagrunner][dagrunner_restart]")
std::string goodPrefix = "rec_error_";
std::string badPrefix = "noexist/rec_error_";
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
R"({"A": {"job": {"command": ["/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
badPrefix +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
badPrefix + R"(C"]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -179,7 +179,7 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string generatorOutput =
R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
fs::path ofn = fs::current_path() / "generator_test_output.json";
std::ofstream ofh{ofn};
ofh << generatorOutput << std::endl;
@@ -188,10 +188,10 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
daggy::TimePoint globalStartTime = daggy::Clock::now();
std::stringstream jsonTasks;
jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", )"
<< R"({ "A": { "job": {"command": [ "/bin/cat", )"
<< std::quoted(ofn.string())
<< R"(]}, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
<< R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2);

14
libdaggy/tests/unit_executor_forkingexecutor.cpp
View File

@@ -18,7 +18,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}};
"/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -32,7 +32,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Simple Run using commandString")
{
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}};
daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -100,7 +100,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}};
"/bin/sleep", "30"}}}};
auto start = daggy::Clock::now();
auto recFuture = ex.execute(0, "command", task);
@@ -120,8 +120,8 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
@@ -130,7 +130,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
"/bin/cat", bigFile}}}};
auto recFuture = ex.execute(0, "command", task);
auto rec = recFuture->get();
@@ -147,7 +147,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params);

4
libdaggy/tests/unit_executor_noopexecutor.cpp
View File

@@ -18,7 +18,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::NoopTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}};
"/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -34,7 +34,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params);

16
libdaggy/tests/unit_executor_slurmexecutor.cpp
View File

@@ -43,7 +43,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run")
{
daggy::Task task{.job{
{"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}}}};
{"command", std::vector<std::string>{"/bin/echo", "abc", "123"}}}};
task.job.merge(defaultJobValues);
@@ -114,7 +114,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run using commandString")
{
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}};
daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
task.job.merge(defaultJobValues);
REQUIRE(ex.validateTaskParameters(task.job));
@@ -131,7 +131,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/expr", "1", "+", "+"}}}};
"/bin/expr", "1", "+", "+"}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -146,7 +146,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}};
"/bin/sleep", "30"}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -161,8 +161,8 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
@@ -171,7 +171,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
"/bin/cat", bigFile}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -190,7 +190,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
auto result = daggy::expandTaskSet(tasks, ex, params);

10
libdaggy/tests/unit_server.cpp
View File

@@ -163,8 +163,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
"tag": "unit_server",
"parameters": { "FILE": [ "A", "B" ] },
"tasks": {
"touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]} },
"cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ]},
"touch": { "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]} },
"cat": { "job": { "command": [ "/bin/cat", "dagrun_A", "dagrun_B" ]},
"parents": [ "touch" ]
}
}
@@ -293,9 +293,9 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
std::string dagRunJSON = R"({
"tag": "unit_server",
"tasks": {
"touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} }
"touch_A": { "job": { "command": [ "/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/bin/touch", "resume_touch_c" ]} }
}
})";

7
libdaggy/tests/unit_threadpool.cpp
View File

@@ -4,6 +4,7 @@
#include "daggy/ThreadPool.hpp"
using namespace std::chrono_literals;
using namespace daggy;
TEST_CASE("threadpool", "[threadpool]")
@@ -36,8 +37,12 @@ TEST_CASE("threadpool", "[threadpool]")
cnt++;
return;
}));
for (auto &r : res)
for (auto &r : res) {
while (! r->ready()) {
std::this_thread::sleep_for(150ms);
}
r->get();
}
REQUIRE(cnt == 100);
}
}

12
libdaggy/tests/unit_utilities.cpp
View File

@@ -41,7 +41,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}",
std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}",
"{{TYPE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
@@ -53,7 +53,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo"};
std::vector<std::string> cmd{"/bin/echo"};
auto allCommands = daggy::interpolateValues(cmd, params);
REQUIRE(allCommands.size() == 1);
@@ -64,7 +64,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"};
std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -76,7 +76,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"};
std::vector<std::string> cmd{"/bin/touch", "{{DATE}}_{{SOURCE}}"};
auto result = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -93,9 +93,9 @@ TEST_CASE("expand_taskset", "[utilities][expand_taskset]")
std::stringstream jsonTasks;
jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", "/etc/passwd"]} )"
<< R"({ "A": { "job": {"command": [ "/bin/cat", "/etc/passwd"]} )"
<< R"(, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
<< R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2);