From f1479a72d92046f0d8d5d4b9e31954281edba543 Mon Sep 17 00:00:00 2001 From: Ian Roddis Date: Tue, 4 Jan 2022 17:03:11 -0400 Subject: [PATCH] Fixing test cases on daggyr for new polling --- daggyd/libdaggyd/src/Server.cpp | 9 +++- daggyr/tests/unit_server.cpp | 75 +++++++++++++++------------------ examples/sample_dag.json | 50 ++++------------------ 3 files changed, 52 insertions(+), 82 deletions(-) diff --git a/daggyd/libdaggyd/src/Server.cpp b/daggyd/libdaggyd/src/Server.cpp index c6a2fa0..7e156ff 100644 --- a/daggyd/libdaggyd/src/Server.cpp +++ b/daggyd/libdaggyd/src/Server.cpp @@ -180,7 +180,14 @@ namespace daggy::daggyd { if (!handleAuth(request)) return; - auto dagSpec = dagFromJSON(request.body()); + DAGSpec dagSpec; + try { + dagSpec = dagFromJSON(request.body()); + } + catch (std::runtime_error &e) { + REQ_RESPONSE(Not_Acceptable, e.what()); + } + dagSpec.tasks = expandTaskSet(dagSpec.tasks, executor_, dagSpec.taskConfig.variables); diff --git a/daggyr/tests/unit_server.cpp b/daggyr/tests/unit_server.cpp index 4d598ee..019fbb8 100644 --- a/daggyr/tests/unit_server.cpp +++ b/daggyr/tests/unit_server.cpp @@ -36,12 +36,6 @@ TEST_CASE("rest_endpoint", "[server_basic]") REQUIRE(response.code == HTTPCode::Ok); } - SECTION("Querying a non-existent task should yield a 404") - { - auto response = HTTP_REQUEST(baseURL + "/v1/task/100/sample_name"); - REQUIRE(response.code == HTTPCode::Not_Found); - } - SECTION("Task Missing Cores should Fail") { std::string taskSpec = @@ -75,22 +69,24 @@ TEST_CASE("rest_endpoint", "[server_basic]") } while (true) { - auto [code, doc] = JSON_HTTP_REQUEST(baseURL + "/v1/task/0/sample_task"); - REQUIRE(doc.IsObject()); - REQUIRE(doc.HasMember("state")); + std::this_thread::sleep_for(250ms); + auto [code, doc] = JSON_HTTP_REQUEST(baseURL + "/v1/poll"); + REQUIRE(doc.IsArray()); + if (doc.Size() == 0) + continue; - std::string state = doc["state"].GetString(); - if (state != "COMPLETED") { - std::this_thread::sleep_for(250ms); - } - else { - REQUIRE(doc.HasMember("attempt")); - auto attempt = attemptRecordFromJSON(doc["attempt"]); + const auto &task = doc[0]; + REQUIRE(task.HasMember("state")); + std::string state = task["state"].GetString(); + if (state != "COMPLETED") + continue; - REQUIRE(attempt.rc == 0); - REQUIRE(attempt.outputLog == "hello world\n"); - break; - } + REQUIRE(task.HasMember("attempt")); + auto attempt = attemptRecordFromJSON(task["attempt"]); + + REQUIRE(attempt.rc == 0); + REQUIRE(attempt.outputLog == "hello world\n"); + break; } } @@ -131,10 +127,11 @@ TEST_CASE("rest_endpoint", "[server_basic]") // Ensure the current job is running { - auto [code, doc] = JSON_HTTP_REQUEST(baseURL + "/v1/task/0/sample_task"); - REQUIRE(doc.IsObject()); - REQUIRE(doc.HasMember("state")); - REQUIRE(doc["state"] != "COMPLETED"); + auto [code, doc] = JSON_HTTP_REQUEST(baseURL + "/v1/poll"); + REQUIRE(doc.IsArray()); + REQUIRE(doc.Size() > 0); + REQUIRE(doc[0].HasMember("state")); + REQUIRE(doc[0]["state"] != "COMPLETED"); } // Stop it @@ -146,25 +143,23 @@ TEST_CASE("rest_endpoint", "[server_basic]") // Grab it and ensure it was killed while (true) { - auto response = HTTP_REQUEST(baseURL + "/v1/task/0/sample_task"); + auto [code, doc] = JSON_HTTP_REQUEST(baseURL + "/v1/poll"); + REQUIRE(code == HTTPCode::Ok); - REQUIRE(response.code == HTTPCode::Ok); - rj::Document doc; - daggy::checkRJParse(doc.Parse(response.body.c_str())); - REQUIRE(doc.IsObject()); - REQUIRE(doc.HasMember("state")); + REQUIRE(doc.IsArray()); + if (doc.Size() == 0) + continue; - std::string state = doc["state"].GetString(); - if (state != "COMPLETED") { - std::this_thread::sleep_for(250ms); - } - else { - REQUIRE(doc.HasMember("attempt")); - auto attempt = attemptRecordFromJSON(doc["attempt"]); + const auto &task = doc[0]; + REQUIRE(task.HasMember("state")); + std::string state = task["state"].GetString(); + if (state != "COMPLETED") + continue; - REQUIRE(attempt.rc != 0); - break; - } + REQUIRE(task.HasMember("attempt")); + auto attempt = attemptRecordFromJSON(task["attempt"]); + REQUIRE(attempt.rc != 0); + break; } } diff --git a/examples/sample_dag.json b/examples/sample_dag.json index 251b400..5327e08 100644 --- a/examples/sample_dag.json +++ b/examples/sample_dag.json @@ -1,43 +1,11 @@ { - "parameters": { - "SOURCE": [ - "a", - "b", - "c" - ], - "DATE": [ - "2021-01-01", - "2021-01-02" - ] - }, - "tasks": [ - { - "name": "pull_data_a", - "max_retries": 3, - "retry_interval_seconds": 600, - "if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_A", - "command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_A", - "verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_A", - "timeout_seconds": 30, - "children": [ - "merge_data" - ] - }, - { - "name": "pull_data_b", - "max_retries": 3, - "retry_interval_seconds": 600, - "if": "/path/to/should_pull.sh --date {{DATE}} --source {{SOURCE}}_B", - "command": "/path/to/pull.sh --date {{DATE}} --source {{SOURCE}}_B", - "verification_command": "/path/to/pull_verify.sh --date {{DATE}} --source {{SOURCE}}_B", - "timeout_seconds": 30, - "children": [ - "merge_data" - ] - }, - { - "name": "merge_data", - "command": "/path/to/merge.sh --left {{SOURCE}}_A --right {{SOURCE}}_B" - } - ] + "tag": "testdag", + "tasks": { + "A": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "b", "c", "d" ]}, + "b": { "job": { "command": [ "/usr/bin/sleep", "2" ], "cores": "1", "memoryMB": "100" }, "children": [ "d" ]}, + "c": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "e", "f" ]}, + "d": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "f" ]}, + "e": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }, "children": [ "f" ]}, + "f": { "job": { "command": [ "/usr/bin/echo", "hello" ], "cores": "1", "memoryMB": "100" }} + } }