Small fixes to test cases

This commit is contained in:
Ian Roddis
2022-01-28 10:23:21 -04:00
parent c09d5b7d8b
commit c9820c19a0
11 changed files with 73 additions and 68 deletions

View File

@@ -91,7 +91,7 @@ Below is an example DAG Run submission:
"task_one": {
"job": {
"command": [
"/usr/bin/touch",
"/bin/touch",
"/tmp/somefile"
]
},
@@ -101,7 +101,7 @@ Below is an example DAG Run submission:
"task_two": {
"job": {
"command": [
"/usr/bin/touch",
"/bin/touch",
"/tmp/someotherfile"
]
},
@@ -133,7 +133,7 @@ For instance:
"task_one": {
"job": {
"command": [
"/usr/bin/touch",
"/bin/touch",
"{{DIRECTORY}}/{{FILE}}"
]
},
@@ -167,7 +167,7 @@ Example:
"populate_inputs": {
"job": {
"command": [
"/usr/bin/touch",
"/bin/touch",
"{{DIRECTORY}}/{{FILE}}"
]
}
@@ -346,8 +346,8 @@ and capture output.
| Field | Sample | Description |
|---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| command | `[ "/bin/echo", "param1" ]` | The command to run |
| commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
Slurm Executor (SlurmTaskExecutor)
@@ -361,8 +361,8 @@ Required `job` config values:
| Field | Sample | Description |
|---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run on a slurm host |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| command | `[ "/bin/echo", "param1" ]` | The command to run on a slurm host |
| commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
| minCPUs | `"1"` | Minimum number of CPUs required |
| minMemoryMB | `"1"` | Minimum memory required, in MB |
@@ -394,8 +394,8 @@ Enforcement via cgroups is planned.
| Field | Sample | Description |
|---------|--------|--------------|
| command | `[ "/usr/bin/echo", "param1" ]` | The command to run |
| commandString | `"/usr/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| command | `[ "/bin/echo", "param1" ]` | The command to run |
| commandString | `"/bin/echo param1"` | The command to run as a string. Quoted args are properly handled. |
| environment | `[ "DATE=2021-05-03" ]` | Environment variables to set for script |
| cores | "1" | Number of cores required by the task |
| memoryMB | "100" | Amount of memory (RSS) required by the task, in MB |

View File

@@ -51,8 +51,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
"tag": "unit_server",
"parameters": { "FILE": [ "A", "B" ] },
"tasks": {
"touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ], "environment": []} },
"cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ], "environment": []},
"touch": { "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ], "environment": []} },
"cat": { "job": { "command": [ "/bin/cat", "dagrun_A", "dagrun_B" ], "environment": []},
"parents": [ "touch" ]
}
}
@@ -182,9 +182,9 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
std::string dagRunJSON = R"({
"tag": "unit_server",
"tasks": {
"touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} }
"touch_A": { "job": { "command": [ "/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/bin/touch", "resume_touch_c" ]} }
}
})";

View File

@@ -39,7 +39,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task Missing Cores should Fail")
{
std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]}, "memoryMB": 100 })";
R"({ "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]}, "memoryMB": 100 })";
auto response =
HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST");
@@ -49,7 +49,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task Missing MemoryMB should Fail")
{
std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]}, "cores": 100 })";
R"({ "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]}, "cores": 100 })";
auto response =
HTTP_REQUEST(baseURL + "/v1/task/0/sample_task", taskSpec, "POST");
@@ -59,7 +59,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task submission and get result")
{
std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/echo", "hello", "world" ], "cores": "1", "memoryMB": "100" } })";
R"({ "job": { "command": [ "/bin/echo", "hello", "world" ], "cores": "1", "memoryMB": "100" } })";
// Submit
{
@@ -93,7 +93,7 @@ TEST_CASE("rest_endpoint", "[server_basic]")
SECTION("Task capacity changes")
{
std::string taskSpec =
R"({ "job": { "command": [ "/usr/bin/sleep", "5" ], "cores": "1", "memoryMB": "100" } })";
R"({ "job": { "command": [ "/bin/sleep", "5" ], "cores": "1", "memoryMB": "100" } })";
auto getCapacity = [&]() -> daggy::executors::task::daggy_runner::Capacity {
daggy::executors::task::daggy_runner::Capacity cap;

View File

@@ -10,7 +10,7 @@
"A": {
"job": {
"command": [
"/usr/bin/echo",
"/bin/echo",
"A",
"{{DATE}}"
]
@@ -24,7 +24,7 @@
"b": {
"job": {
"command": [
"/usr/bin/sleep",
"/bin/sleep",
"2"
]
},
@@ -35,7 +35,7 @@
"c": {
"job": {
"command": [
"/usr/bin/echo",
"/bin/echo",
"C",
"{{DATE}}"
]
@@ -48,7 +48,7 @@
"d": {
"job": {
"command": [
"/usr/bin/echo",
"/bin/echo",
"D",
"{{DATE}}"
]
@@ -60,7 +60,7 @@
"e": {
"job": {
"command": [
"/usr/bin/echo",
"/bin/echo",
"E",
"{{DATE}}"
]
@@ -72,7 +72,7 @@
"f": {
"job": {
"command": [
"/usr/bin/echo",
"/bin/echo",
"F",
"{{DATE}}"
]

View File

@@ -24,11 +24,11 @@ TEST_CASE("dagrunner", "[dagrunner][dagrunner_order_preservation]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string taskJSON = R"({
"A": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
"D": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}, "children": [ "E"]},
"E": {"job": {"command": ["/usr/bin/touch", "{{DATE}}"]}}
"A": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "B","D" ]},
"B": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "C","D","E" ]},
"C": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "D"]},
"D": {"job": {"command": ["/bin/touch", "{{DATE}}"]}, "children": [ "E"]},
"E": {"job": {"command": ["/bin/touch", "{{DATE}}"]}}
})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex,
@@ -83,10 +83,10 @@ TEST_CASE("DAGRunner simple execution", "[dagrunner][dagrunner_simple]")
std::unordered_map<std::string, std::string> files{
{"A", prefix + "_A"}, {"B", prefix + "_B"}, {"C", prefix + "_C"}};
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
R"({"A": {"job": {"command": ["/bin/touch", ")" + files.at("A") +
R"("]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
files.at("B") +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
R"("]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
files.at("C") + R"("]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -132,10 +132,10 @@ TEST_CASE("DAG Runner Restart old DAG", "[dagrunner][dagrunner_restart]")
std::string goodPrefix = "rec_error_";
std::string badPrefix = "noexist/rec_error_";
std::string taskJSON =
R"({"A": {"job": {"command": ["/usr/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/usr/bin/touch", ")" +
R"({"A": {"job": {"command": ["/bin/touch", ")" + goodPrefix +
R"(A"]}, "children": ["C"]}, "B": {"job": {"command": ["/bin/touch", ")" +
badPrefix +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/usr/bin/touch", ")" +
R"(B"]}, "children": ["C"]}, "C": {"job": {"command": ["/bin/touch", ")" +
badPrefix + R"(C"]}}})";
dagSpec.tasks = expandTaskSet(daggy::tasksFromJSON(taskJSON), ex);
auto dag = daggy::buildDAGFromTasks(dagSpec.tasks);
@@ -179,7 +179,7 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
dagSpec.taskConfig.variables = daggy::configFromJSON(testParams);
std::string generatorOutput =
R"({"B": {"job": {"command": ["/usr/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "-e", "{{DATE}}"]}, "children": ["C"]}})";
fs::path ofn = fs::current_path() / "generator_test_output.json";
std::ofstream ofh{ofn};
ofh << generatorOutput << std::endl;
@@ -188,10 +188,10 @@ TEST_CASE("DAG Runner Generator Tasks", "[dagrunner][dagrunner_generator]")
daggy::TimePoint globalStartTime = daggy::Clock::now();
std::stringstream jsonTasks;
jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", )"
<< R"({ "A": { "job": {"command": [ "/bin/cat", )"
<< std::quoted(ofn.string())
<< R"(]}, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
<< R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2);

View File

@@ -18,7 +18,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}};
"/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -32,7 +32,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Simple Run using commandString")
{
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}};
daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -100,7 +100,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}};
"/bin/sleep", "30"}}}};
auto start = daggy::Clock::now();
auto recFuture = ex.execute(0, "command", task);
@@ -120,8 +120,8 @@ TEST_CASE("forking_executor", "[forking_executor]")
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
@@ -130,7 +130,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
daggy::Task task{
.job{{"command", daggy::executors::task::ForkingTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
"/bin/cat", bigFile}}}};
auto recFuture = ex.execute(0, "command", task);
auto rec = recFuture->get();
@@ -147,7 +147,7 @@ TEST_CASE("forking_executor", "[forking_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params);

View File

@@ -18,7 +18,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::NoopTaskExecutor::Command{
"/usr/bin/echo", "abc", "123"}}}};
"/bin/echo", "abc", "123"}}}};
REQUIRE(ex.validateTaskParameters(task.job));
@@ -34,7 +34,7 @@ TEST_CASE("noop_executor", "[noop_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"], "environment": [ "TEST={{DATE}}"] }, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON);
auto result = daggy::expandTaskSet(tasks, ex, params);

View File

@@ -43,7 +43,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run")
{
daggy::Task task{.job{
{"command", std::vector<std::string>{"/usr/bin/echo", "abc", "123"}}}};
{"command", std::vector<std::string>{"/bin/echo", "abc", "123"}}}};
task.job.merge(defaultJobValues);
@@ -114,7 +114,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Simple Run using commandString")
{
daggy::Task task{.job{{"commandString", R"(/usr/bin/echo "abc 123")"}}};
daggy::Task task{.job{{"commandString", R"(/bin/echo "abc 123")"}}};
task.job.merge(defaultJobValues);
REQUIRE(ex.validateTaskParameters(task.job));
@@ -131,7 +131,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/expr", "1", "+", "+"}}}};
"/bin/expr", "1", "+", "+"}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -146,7 +146,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
{
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/sleep", "30"}}}};
"/bin/sleep", "30"}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -161,8 +161,8 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
SECTION("Large Output")
{
const std::vector<std::string> BIG_FILES{"/usr/share/dict/linux.words",
"/usr/share/dict/cracklib-small",
const std::vector<std::string> BIG_FILES{"/share/dict/linux.words",
"/share/dict/cracklib-small",
"/etc/ssh/moduli"};
for (const auto &bigFile : BIG_FILES) {
@@ -171,7 +171,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
daggy::Task task{
.job{{"command", daggy::executors::task::SlurmTaskExecutor::Command{
"/usr/bin/cat", bigFile}}}};
"/bin/cat", bigFile}}}};
task.job.merge(defaultJobValues);
auto recFuture = ex.execute(0, "command", task);
@@ -190,7 +190,7 @@ TEST_CASE("slurm_execution", "[slurm][slurm_executor]")
auto params = daggy::configFromJSON(testParams);
std::string taskJSON =
R"({"B": {"job": {"command": ["/usr/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
R"({"B": {"job": {"command": ["/bin/echo", "{{DATE}}"]}, "children": ["C"]}})";
auto tasks = daggy::tasksFromJSON(taskJSON, defaultJobValues);
auto result = daggy::expandTaskSet(tasks, ex, params);

View File

@@ -163,8 +163,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
"tag": "unit_server",
"parameters": { "FILE": [ "A", "B" ] },
"tasks": {
"touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]} },
"cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ]},
"touch": { "job": { "command": [ "/bin/touch", "dagrun_{{FILE}}" ]} },
"cat": { "job": { "command": [ "/bin/cat", "dagrun_A", "dagrun_B" ]},
"parents": [ "touch" ]
}
}
@@ -293,9 +293,9 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
std::string dagRunJSON = R"({
"tag": "unit_server",
"tasks": {
"touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} }
"touch_A": { "job": { "command": [ "/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] },
"sleep_B": { "job": { "command": [ "/bin/sleep", "3" ]}, "children": ["touch_C"] },
"touch_C": { "job": { "command": [ "/bin/touch", "resume_touch_c" ]} }
}
})";

View File

@@ -4,6 +4,7 @@
#include "daggy/ThreadPool.hpp"
using namespace std::chrono_literals;
using namespace daggy;
TEST_CASE("threadpool", "[threadpool]")
@@ -36,8 +37,12 @@ TEST_CASE("threadpool", "[threadpool]")
cnt++;
return;
}));
for (auto &r : res)
for (auto &r : res) {
while (! r->ready()) {
std::this_thread::sleep_for(150ms);
}
r->get();
}
REQUIRE(cnt == 100);
}
}

View File

@@ -41,7 +41,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}",
std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}",
"{{TYPE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
@@ -53,7 +53,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo"};
std::vector<std::string> cmd{"/bin/echo"};
auto allCommands = daggy::interpolateValues(cmd, params);
REQUIRE(allCommands.size() == 1);
@@ -64,7 +64,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"};
std::vector<std::string> cmd{"/bin/echo", "{{DATE}}", "{{SOURCE}}"};
auto allCommands = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -76,7 +76,7 @@ TEST_CASE("string_expansion", "[utilities][parameter_expansion]")
std::string testParams{
R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"};
auto params = daggy::configFromJSON(testParams);
std::vector<std::string> cmd{"/usr/bin/touch", "{{DATE}}_{{SOURCE}}"};
std::vector<std::string> cmd{"/bin/touch", "{{DATE}}_{{SOURCE}}"};
auto result = daggy::interpolateValues(cmd, params);
// TYPE isn't used, so it's just |DATE| * |SOURCE|
@@ -93,9 +93,9 @@ TEST_CASE("expand_taskset", "[utilities][expand_taskset]")
std::stringstream jsonTasks;
jsonTasks
<< R"({ "A": { "job": {"command": [ "/usr/bin/cat", "/etc/passwd"]} )"
<< R"({ "A": { "job": {"command": [ "/bin/cat", "/etc/passwd"]} )"
<< R"(, "children": ["C"], "isGenerator": true},)"
<< R"("C": { "job": {"command": [ "/usr/bin/echo", "hello!"]} } })";
<< R"("C": { "job": {"command": [ "/bin/echo", "hello!"]} } })";
dagSpec.tasks = daggy::tasksFromJSON(jsonTasks.str());
REQUIRE(dagSpec.tasks.size() == 2);