#include #include #include #include #include #include #include #include "daggy/Utilities.hpp" #include "daggy/Serialization.hpp" #include "daggy/executors/task/ForkingTaskExecutor.hpp" #include "daggy/loggers/dag_run/OStreamLogger.hpp" namespace fs = std::filesystem; TEST_CASE("String Utilities", "[utilities_string]") { std::string test = "/this/is/{{A}}/test/{{A}}"; auto res = daggy::globalSub(test, "{{A}}", "hello"); REQUIRE(res == "/this/is/hello/test/hello"); } TEST_CASE("Parameter Expansion", "[utilities_parameter_expansion]") { SECTION("Basic expansion") { std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; auto params = daggy::parametersFromJSON(testParams); std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}", "{{TYPE}}"}; auto allCommands = daggy::expandCommands(cmd, params); REQUIRE(allCommands.size() == 6); } SECTION("Skip over unused parameters") { std::string testParams{R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": "name", "TYPE": ["a", "b", "c"]})"}; auto params = daggy::parametersFromJSON(testParams); std::vector cmd{"/usr/bin/echo", "{{DATE}}", "{{SOURCE}}"}; auto allCommands = daggy::expandCommands(cmd, params); // TYPE isn't used, so it's just |DATE| * |SOURCE| REQUIRE(allCommands.size() == 2); } SECTION("Expand within a command part") { std::string testParams{ R"({"DATE": ["2021-05-06", "2021-05-07" ], "SOURCE": ["A", "B"], "TYPE": ["a", "b", "c"]})"}; auto params = daggy::parametersFromJSON(testParams); std::vector cmd{"/usr/bin/touch", "/tmp/{{DATE}}_{{SOURCE}}"}; auto result = daggy::expandCommands(cmd, params); // TYPE isn't used, so it's just |DATE| * |SOURCE| REQUIRE(result.size() == 4); } } TEST_CASE("DAG Runner", "[utilities_dag_runner]") { daggy::executors::task::ForkingTaskExecutor ex(10); std::stringstream ss; daggy::loggers::dag_run::OStreamLogger logger(ss); SECTION("Simple execution") { std::string prefix = "/tmp/asdlk_"; std::string taskJSON = R"([{"name": "A", "command": ["/usr/bin/touch", ")" + prefix + R"(A"], "children": ["C"]}, {"name": "B", "command": ["/usr/bin/touch", ")" + prefix + R"(B"], "children": ["C"]}, {"name": "C", "command": ["/usr/bin/touch", ")" + prefix + R"(C"]}])"; auto tasks = daggy::tasksFromJSON(taskJSON); auto dag = daggy::buildDAGFromTasks(tasks); auto runID = logger.startDAGRun("test_run", tasks); auto endDAG = daggy::runDAG(runID, tasks, ex, logger, dag); REQUIRE(endDAG.allVisited()); std::vector letters{"A", "B", "C"}; for (const auto &letter : letters) { fs::path file{prefix + letter}; REQUIRE(fs::exists(file)); fs::remove(file); } // Get the DAG Run Attempts auto record = logger.getDAGRun(runID); for (const auto &attempts : record.taskAttempts) { REQUIRE(attempts.size() == 1); REQUIRE(attempts.front().rc == 0); } } }