diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f9498..99571ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,5 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) enable_testing() -add_subdirectory(daggy) -add_subdirectory(tests) -add_subdirectory(utils) +add_subdirectory(libdaggy) +add_subdirectory(daggyd) diff --git a/daggyd/CMakeLists.txt b/daggyd/CMakeLists.txt new file mode 100644 index 0000000..8e16b36 --- /dev/null +++ b/daggyd/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(libdaggyd) +add_subdirectory(daggyd) +add_subdirectory(tests) diff --git a/daggyd/README.md b/daggyd/README.md new file mode 100644 index 0000000..a61437d --- /dev/null +++ b/daggyd/README.md @@ -0,0 +1,127 @@ +# Daggy Daemon + +`daggyd` is the REST server process that handles receiving and running DAG specs. + +# Running it + +```bash +daggyd # That's it, will listen on 127.0.0.1:2503 , and run with a local executor +daggyd -d # Daemonize + +daggyd --config FILE # Run with a config file +``` + +# Config Files + +```json +{ + "web-threads": 50, + "dag-threads": 50, + "port": 2503, + "ip": "localhost", + "logger": { + "name": "LoggerName", + "config": { + ... + } + }, + "executor": { + "name": "ExecutorName" + "config": { + ... + } + } +} +``` + +## Loggers + +### OStreamLogger + +OStreamLogger doesn't persist data, but can write even updates to a file or +stdout. + +The config for OStreamLogger looks like this: + +``` +{ + ... + "logger": { + "name": "OStreamLogger", + "config": { + "file": "/path/to/file" + } + } + ... +} +``` + +If `file` is equal to `"-"`, then the logger will print events to stdout. This configuration +is the default if no logger is specified at all. + +### RedisLogger + +RedisLogger stores state in a [Redis](https://redis.io) instance. + +The config for OStreamLogger looks like this (along with default values): + +``` +{ + ... + "logger": { + "name": "RedisLogger", + "config": { + "prefix": "daggy", + "host": "localhost", + "port": 6379 + } + } + ... +} +``` + +The `prefix` attribute is used to distinguish daggy instances. All keys will be prefixed with +the value of `prefix`. + + +## Executors + +### ForkingTaskExecutor + +ForkingTaskExecutor does pretty much what the name implies: it will execute tasks by +forking on the local machine. + +It's config with default values looks like: + +``` +{ + ... + "executor": { + "name": "ForkingTaskExecutor", + "config": { + "threads": 10 + } + } + ... +} +``` + +If no executor is sepcified in the config, this is the executor used. + +### SlurmTaskExecutor + +The SlurmTaskExecutor will execute tasks on a [slurm](https://slurm.schedmd.com) cluster. It relies +on the slurm config to manage any parallelism limits and quotas. + +It's config with default values looks like: + +``` +{ + ... + "executor": { + "name": "ForkingTaskExecutor", + "config": { } + } + ... +} +``` diff --git a/daggyd/daggyd/CMakeLists.txt b/daggyd/daggyd/CMakeLists.txt new file mode 100644 index 0000000..60bb9bb --- /dev/null +++ b/daggyd/daggyd/CMakeLists.txt @@ -0,0 +1,4 @@ +project(daggyd) +file(GLOB SOURCES daggyd.cpp) +add_executable(${PROJECT_NAME} ${SOURCES}) +target_link_libraries(${PROJECT_NAME} argparse libdaggyd libdaggy) diff --git a/utils/daggyd/daggyd.cpp b/daggyd/daggyd/daggyd.cpp similarity index 96% rename from utils/daggyd/daggyd.cpp rename to daggyd/daggyd/daggyd.cpp index f0c77a9..e884960 100644 --- a/utils/daggyd/daggyd.cpp +++ b/daggyd/daggyd/daggyd.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include @@ -14,12 +14,11 @@ #include // Add loggers here +#include +#include #include #include -#include "daggy/executors/task/TaskExecutor.hpp" -#include "daggy/loggers/dag_run/DAGRunLogger.hpp" - namespace rj = rapidjson; static std::atomic running{true}; @@ -126,6 +125,10 @@ std::unique_ptr loggerFactory(const rj::Value &config) const auto &logConfig = logConf["config"]; if (name == "OStreamLogger") { if (logConfig.HasMember("file")) { + std::string fn = logConfig["file"].GetString(); + if (fn == "-") + return std::make_unique(std::cout); + std::ofstream ofh(logConfig["file"].GetString()); return std::make_unique(ofh); } diff --git a/daggyd/libdaggyd/CMakeLists.txt b/daggyd/libdaggyd/CMakeLists.txt new file mode 100644 index 0000000..50edf29 --- /dev/null +++ b/daggyd/libdaggyd/CMakeLists.txt @@ -0,0 +1,8 @@ +project(libdaggyd) + +add_library(${PROJECT_NAME} STATIC) + +target_include_directories(${PROJECT_NAME} PUBLIC include) +target_link_libraries(${PROJECT_NAME} libdaggy) + +add_subdirectory(src) diff --git a/daggy/include/daggy/Server.hpp b/daggyd/libdaggyd/include/daggyd/Server.hpp similarity index 91% rename from daggy/include/daggy/Server.hpp rename to daggyd/libdaggyd/include/daggyd/Server.hpp index 2f6dbc8..572488c 100644 --- a/daggy/include/daggy/Server.hpp +++ b/daggyd/libdaggyd/include/daggyd/Server.hpp @@ -4,13 +4,12 @@ #include #include +#include +#include +#include +#include #include -#include "DAGRunner.hpp" -#include "ThreadPool.hpp" -#include "executors/task/TaskExecutor.hpp" -#include "loggers/dag_run/DAGRunLogger.hpp" - #define DAGGY_REST_HANDLER(func) \ void func(const Pistache::Rest::Request &request, \ Pistache::Http::ResponseWriter response); diff --git a/daggyd/libdaggyd/src/CMakeLists.txt b/daggyd/libdaggyd/src/CMakeLists.txt new file mode 100644 index 0000000..9f09ece --- /dev/null +++ b/daggyd/libdaggyd/src/CMakeLists.txt @@ -0,0 +1,3 @@ +target_sources(${PROJECT_NAME} PRIVATE + Server.cpp +) diff --git a/daggy/src/Server.cpp b/daggyd/libdaggyd/src/Server.cpp similarity index 99% rename from daggy/src/Server.cpp rename to daggyd/libdaggyd/src/Server.cpp index e76509b..6b5462f 100644 --- a/daggy/src/Server.cpp +++ b/daggyd/libdaggyd/src/Server.cpp @@ -1,8 +1,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/daggyd/tests/CMakeLists.txt b/daggyd/tests/CMakeLists.txt new file mode 100644 index 0000000..8c328aa --- /dev/null +++ b/daggyd/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +project(daggyd_tests) + +add_executable(${PROJECT_NAME} main.cpp + # unit tests + unit_server.cpp + ) +target_link_libraries(${PROJECT_NAME} libdaggyd libdaggy stdc++fs Catch2::Catch2 curl) + +add_test(${PROJECT_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}) diff --git a/tests/main.cpp b/daggyd/tests/main.cpp similarity index 100% rename from tests/main.cpp rename to daggyd/tests/main.cpp diff --git a/daggyd/tests/unit_server.cpp b/daggyd/tests/unit_server.cpp new file mode 100644 index 0000000..1049c41 --- /dev/null +++ b/daggyd/tests/unit_server.cpp @@ -0,0 +1,382 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rj = rapidjson; + +using namespace daggy; + +#ifdef DEBUG_HTTP +static int my_trace(CURL *handle, curl_infotype type, char *data, size_t size, + void *userp) +{ + const char *text; + (void)handle; /* prevent compiler warning */ + (void)userp; + + switch (type) { + case CURLINFO_TEXT: + fprintf(stderr, "== Info: %s", data); + default: /* in case a new one is introduced to shock us */ + return 0; + + case CURLINFO_HEADER_OUT: + text = "=> Send header"; + break; + case CURLINFO_DATA_OUT: + text = "=> Send data"; + break; + case CURLINFO_SSL_DATA_OUT: + text = "=> Send SSL data"; + break; + case CURLINFO_HEADER_IN: + text = "<= Recv header"; + break; + case CURLINFO_DATA_IN: + text = "<= Recv data"; + break; + case CURLINFO_SSL_DATA_IN: + text = "<= Recv SSL data"; + break; + } + + std::cerr << "\n================== " << text + << " ==================" << std::endl + << data << std::endl; + return 0; +} +#endif + +enum HTTPCode : long +{ + Ok = 200, + Not_Found = 404 +}; + +struct HTTPResponse +{ + HTTPCode code; + std::string body; +}; + +uint curlWriter(char *in, uint size, uint nmemb, std::stringstream *out) +{ + uint r; + r = size * nmemb; + out->write(in, r); + return r; +} + +HTTPResponse REQUEST(const std::string &url, const std::string &payload = "", + const std::string &method = "GET") +{ + HTTPResponse response; + + CURL *curl; + CURLcode res; + struct curl_slist *headers = NULL; + + curl_global_init(CURL_GLOBAL_ALL); + + curl = curl_easy_init(); + if (curl) { + std::stringstream buffer; + +#ifdef DEBUG_HTTP + curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, my_trace); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); +#endif + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriter); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); + + if (!payload.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, payload.size()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); + headers = curl_slist_append(headers, "Content-Type: Application/Json"); + } + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); + headers = curl_slist_append(headers, "Expect:"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + curl_easy_cleanup(curl); + throw std::runtime_error(std::string{"CURL Failed: "} + + curl_easy_strerror(res)); + } + curl_easy_cleanup(curl); + + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response.code); + response.body = buffer.str(); + } + + curl_global_cleanup(); + + return response; +} + +TEST_CASE("rest_endpoint", "[server_basic]") +{ + std::stringstream ss; + daggy::executors::task::ForkingTaskExecutor executor(10); + daggy::loggers::dag_run::OStreamLogger logger(ss); + Pistache::Address listenSpec("localhost", Pistache::Port(0)); + + const size_t nDAGRunners = 10, nWebThreads = 10; + + daggy::Server server(listenSpec, logger, executor, nDAGRunners); + server.init(nWebThreads); + server.start(); + + const std::string host = "localhost:"; + const std::string baseURL = host + std::to_string(server.getPort()); + + SECTION("Ready Endpoint") + { + auto response = REQUEST(baseURL + "/ready"); + REQUIRE(response.code == HTTPCode::Ok); + } + + SECTION("Querying a non-existent dagrunid should fail ") + { + auto response = REQUEST(baseURL + "/v1/dagrun/100"); + REQUIRE(response.code != HTTPCode::Ok); + } + + SECTION("Simple DAGRun Submission") + { + std::string dagRun = R"({ + "tag": "unit_server", + "parameters": { "FILE": [ "A", "B" ] }, + "tasks": { + "touch": { "job": { "command": [ "/usr/bin/touch", "dagrun_{{FILE}}" ]} }, + "cat": { "job": { "command": [ "/usr/bin/cat", "dagrun_A", "dagrun_B" ]}, + "parents": [ "touch" ] + } + } + })"; + + auto dagSpec = daggy::dagFromJSON(dagRun); + + // Submit, and get the runID + daggy::DAGRunID runID = 0; + { + auto response = REQUEST(baseURL + "/v1/dagrun/", dagRun, "POST"); + REQUIRE(response.code == HTTPCode::Ok); + + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body.c_str())); + REQUIRE(doc.IsObject()); + REQUIRE(doc.HasMember("runID")); + + runID = doc["runID"].GetUint64(); + } + + // Ensure our runID shows up in the list of running DAGs + { + auto response = REQUEST(baseURL + "/v1/dagruns?all=1"); + REQUIRE(response.code == HTTPCode::Ok); + + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body.c_str())); + REQUIRE(doc.IsArray()); + REQUIRE(doc.Size() >= 1); + + // Ensure that our DAG is in the list and matches our given DAGRunID + bool found = false; + const auto &runs = doc.GetArray(); + for (size_t i = 0; i < runs.Size(); ++i) { + const auto &run = runs[i]; + REQUIRE(run.IsObject()); + REQUIRE(run.HasMember("tag")); + REQUIRE(run.HasMember("runID")); + + std::string runName = run["tag"].GetString(); + if (runName == "unit_server") { + REQUIRE(run["runID"].GetUint64() == runID); + found = true; + break; + } + } + REQUIRE(found); + } + + // Ensure we can get one of our tasks + { + auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID) + + "/task/cat_0"); + REQUIRE(response.code == HTTPCode::Ok); + + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body.c_str())); + + REQUIRE_NOTHROW(daggy::taskFromJSON("cat", doc)); + auto task = daggy::taskFromJSON("cat", doc); + + REQUIRE(task == dagSpec.tasks.at("cat")); + } + + // Wait until our DAG is complete + bool complete = true; + for (auto i = 0; i < 10; ++i) { + auto response = REQUEST(baseURL + "/v1/dagrun/" + std::to_string(runID)); + REQUIRE(response.code == HTTPCode::Ok); + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body.c_str())); + REQUIRE(doc.IsObject()); + + REQUIRE(doc.HasMember("taskStates")); + const auto &taskStates = doc["taskStates"].GetObject(); + + size_t nStates = 0; + for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); + ++it) { + nStates++; + } + REQUIRE(nStates == 3); + + complete = true; + for (auto it = taskStates.MemberBegin(); it != taskStates.MemberEnd(); + ++it) { + std::string state = it->value.GetString(); + if (state != "COMPLETED") { + complete = false; + break; + } + } + if (complete) + break; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + REQUIRE(complete); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + for (const auto &pth : std::vector{"dagrun_A", "dagrun_B"}) { + REQUIRE(fs::exists(pth)); + fs::remove(pth); + } + } +} + +TEST_CASE("Server cancels and resumes execution", "[server_resume]") +{ + std::stringstream ss; + daggy::executors::task::ForkingTaskExecutor executor(10); + daggy::loggers::dag_run::OStreamLogger logger(ss); + Pistache::Address listenSpec("localhost", Pistache::Port(0)); + + const size_t nDAGRunners = 10, nWebThreads = 10; + + daggy::Server server(listenSpec, logger, executor, nDAGRunners); + server.init(nWebThreads); + server.start(); + + const std::string host = "localhost:"; + const std::string baseURL = host + std::to_string(server.getPort()); + + SECTION("Cancel / Resume DAGRun") + { + std::string dagRunJSON = R"({ + "tag": "unit_server", + "tasks": { + "touch_A": { "job": { "command": [ "/usr/bin/touch", "resume_touch_a" ]}, "children": ["touch_C"] }, + "sleep_B": { "job": { "command": [ "/usr/bin/sleep", "3" ]}, "children": ["touch_C"] }, + "touch_C": { "job": { "command": [ "/usr/bin/touch", "resume_touch_c" ]} } + } + })"; + + auto dagSpec = daggy::dagFromJSON(dagRunJSON); + + // Submit, and get the runID + daggy::DAGRunID runID; + { + auto response = REQUEST(baseURL + "/v1/dagrun/", dagRunJSON, "POST"); + REQUIRE(response.code == HTTPCode::Ok); + + rj::Document doc; + daggy::checkRJParse(doc.Parse(response.body.c_str())); + REQUIRE(doc.IsObject()); + REQUIRE(doc.HasMember("runID")); + + runID = doc["runID"].GetUint64(); + } + + std::this_thread::sleep_for(1s); + + // Stop the current run + { + auto response = REQUEST( + baseURL + "/v1/dagrun/" + std::to_string(runID) + "/state/KILLED", "", + "PATCH"); + REQUIRE(response.code == HTTPCode::Ok); + REQUIRE(logger.getDAGRunState(runID) == +daggy::RunState::KILLED); + } + + // Verify that the run still exists + { + auto dagRun = logger.getDAGRun(runID); + REQUIRE(dagRun.taskRunStates.at("touch_A_0") == + +daggy::RunState::COMPLETED); + REQUIRE(fs::exists("resume_touch_a")); + + REQUIRE(dagRun.taskRunStates.at("sleep_B_0") == + +daggy::RunState::ERRORED); + REQUIRE(dagRun.taskRunStates.at("touch_C_0") == +daggy::RunState::QUEUED); + } + + // Set the errored task state + { + auto url = baseURL + "/v1/dagrun/" + std::to_string(runID) + + "/task/sleep_B_0/state/QUEUED"; + auto response = REQUEST(url, "", "PATCH"); + REQUIRE(response.code == HTTPCode::Ok); + REQUIRE(logger.getTaskState(runID, "sleep_B_0") == + +daggy::RunState::QUEUED); + } + + // Resume + { + struct stat s; + + lstat("resume_touch_A", &s); + auto preMTime = s.st_mtim.tv_sec; + + auto response = REQUEST( + baseURL + "/v1/dagrun/" + std::to_string(runID) + "/state/QUEUED", "", + "PATCH"); + + // Wait for run to complete + std::this_thread::sleep_for(5s); + REQUIRE(logger.getDAGRunState(runID) == +daggy::RunState::COMPLETED); + + REQUIRE(fs::exists("resume_touch_c")); + REQUIRE(fs::exists("resume_touch_a")); + + for (const auto &[taskName, task] : dagSpec.tasks) { + REQUIRE(logger.getTaskState(runID, taskName + "_0") == + +daggy::RunState::COMPLETED); + } + + // Ensure "touch_A" wasn't run again + lstat("resume_touch_A", &s); + auto postMTime = s.st_mtim.tv_sec; + REQUIRE(preMTime == postMTime); + } + } + + server.shutdown(); +} diff --git a/daggy/CMakeLists.txt b/libdaggy/CMakeLists.txt similarity index 94% rename from daggy/CMakeLists.txt rename to libdaggy/CMakeLists.txt index 2413319..ca0e602 100644 --- a/daggy/CMakeLists.txt +++ b/libdaggy/CMakeLists.txt @@ -14,3 +14,4 @@ target_include_directories(${PROJECT_NAME} PUBLIC include) target_link_libraries(${PROJECT_NAME} pistache pthread rapidjson better-enums) add_subdirectory(src) +add_subdirectory(tests) diff --git a/daggy/include/daggy/DAG.hpp b/libdaggy/include/daggy/DAG.hpp similarity index 100% rename from daggy/include/daggy/DAG.hpp rename to libdaggy/include/daggy/DAG.hpp diff --git a/daggy/include/daggy/DAG.impl.hxx b/libdaggy/include/daggy/DAG.impl.hxx similarity index 100% rename from daggy/include/daggy/DAG.impl.hxx rename to libdaggy/include/daggy/DAG.impl.hxx diff --git a/daggy/include/daggy/DAGRunner.hpp b/libdaggy/include/daggy/DAGRunner.hpp similarity index 100% rename from daggy/include/daggy/DAGRunner.hpp rename to libdaggy/include/daggy/DAGRunner.hpp diff --git a/daggy/include/daggy/Defines.hpp b/libdaggy/include/daggy/Defines.hpp similarity index 100% rename from daggy/include/daggy/Defines.hpp rename to libdaggy/include/daggy/Defines.hpp diff --git a/daggy/include/daggy/Serialization.hpp b/libdaggy/include/daggy/Serialization.hpp similarity index 100% rename from daggy/include/daggy/Serialization.hpp rename to libdaggy/include/daggy/Serialization.hpp diff --git a/daggy/include/daggy/ThreadPool.hpp b/libdaggy/include/daggy/ThreadPool.hpp similarity index 100% rename from daggy/include/daggy/ThreadPool.hpp rename to libdaggy/include/daggy/ThreadPool.hpp diff --git a/daggy/include/daggy/Utilities.hpp b/libdaggy/include/daggy/Utilities.hpp similarity index 100% rename from daggy/include/daggy/Utilities.hpp rename to libdaggy/include/daggy/Utilities.hpp diff --git a/daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp b/libdaggy/include/daggy/executors/task/ForkingTaskExecutor.hpp similarity index 100% rename from daggy/include/daggy/executors/task/ForkingTaskExecutor.hpp rename to libdaggy/include/daggy/executors/task/ForkingTaskExecutor.hpp diff --git a/daggy/include/daggy/executors/task/NoopTaskExecutor.hpp b/libdaggy/include/daggy/executors/task/NoopTaskExecutor.hpp similarity index 100% rename from daggy/include/daggy/executors/task/NoopTaskExecutor.hpp rename to libdaggy/include/daggy/executors/task/NoopTaskExecutor.hpp diff --git a/daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp b/libdaggy/include/daggy/executors/task/SlurmTaskExecutor.hpp similarity index 100% rename from daggy/include/daggy/executors/task/SlurmTaskExecutor.hpp rename to libdaggy/include/daggy/executors/task/SlurmTaskExecutor.hpp diff --git a/daggy/include/daggy/executors/task/TaskExecutor.hpp b/libdaggy/include/daggy/executors/task/TaskExecutor.hpp similarity index 100% rename from daggy/include/daggy/executors/task/TaskExecutor.hpp rename to libdaggy/include/daggy/executors/task/TaskExecutor.hpp diff --git a/daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp b/libdaggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp similarity index 100% rename from daggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp rename to libdaggy/include/daggy/loggers/dag_run/DAGRunLogger.hpp diff --git a/daggy/include/daggy/loggers/dag_run/Defines.hpp b/libdaggy/include/daggy/loggers/dag_run/Defines.hpp similarity index 100% rename from daggy/include/daggy/loggers/dag_run/Defines.hpp rename to libdaggy/include/daggy/loggers/dag_run/Defines.hpp diff --git a/daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp b/libdaggy/include/daggy/loggers/dag_run/OStreamLogger.hpp similarity index 100% rename from daggy/include/daggy/loggers/dag_run/OStreamLogger.hpp rename to libdaggy/include/daggy/loggers/dag_run/OStreamLogger.hpp diff --git a/daggy/include/daggy/loggers/dag_run/RedisHelper.hpp b/libdaggy/include/daggy/loggers/dag_run/RedisHelper.hpp similarity index 100% rename from daggy/include/daggy/loggers/dag_run/RedisHelper.hpp rename to libdaggy/include/daggy/loggers/dag_run/RedisHelper.hpp diff --git a/daggy/include/daggy/loggers/dag_run/RedisLogger.hpp b/libdaggy/include/daggy/loggers/dag_run/RedisLogger.hpp similarity index 100% rename from daggy/include/daggy/loggers/dag_run/RedisLogger.hpp rename to libdaggy/include/daggy/loggers/dag_run/RedisLogger.hpp diff --git a/daggy/src/CMakeLists.txt b/libdaggy/src/CMakeLists.txt similarity index 91% rename from daggy/src/CMakeLists.txt rename to libdaggy/src/CMakeLists.txt index 7dbe518..75f6ac4 100644 --- a/daggy/src/CMakeLists.txt +++ b/libdaggy/src/CMakeLists.txt @@ -1,6 +1,5 @@ target_sources(${PROJECT_NAME} PRIVATE Serialization.cpp - Server.cpp Utilities.cpp DAGRunner.cpp ) diff --git a/daggy/src/DAGRunner.cpp b/libdaggy/src/DAGRunner.cpp similarity index 100% rename from daggy/src/DAGRunner.cpp rename to libdaggy/src/DAGRunner.cpp diff --git a/daggy/src/Serialization.cpp b/libdaggy/src/Serialization.cpp similarity index 100% rename from daggy/src/Serialization.cpp rename to libdaggy/src/Serialization.cpp diff --git a/daggy/src/Utilities.cpp b/libdaggy/src/Utilities.cpp similarity index 100% rename from daggy/src/Utilities.cpp rename to libdaggy/src/Utilities.cpp diff --git a/daggy/src/executors/CMakeLists.txt b/libdaggy/src/executors/CMakeLists.txt similarity index 100% rename from daggy/src/executors/CMakeLists.txt rename to libdaggy/src/executors/CMakeLists.txt diff --git a/daggy/src/executors/task/CMakeLists.txt b/libdaggy/src/executors/task/CMakeLists.txt similarity index 100% rename from daggy/src/executors/task/CMakeLists.txt rename to libdaggy/src/executors/task/CMakeLists.txt diff --git a/daggy/src/executors/task/ForkingTaskExecutor.cpp b/libdaggy/src/executors/task/ForkingTaskExecutor.cpp similarity index 100% rename from daggy/src/executors/task/ForkingTaskExecutor.cpp rename to libdaggy/src/executors/task/ForkingTaskExecutor.cpp diff --git a/daggy/src/executors/task/NoopTaskExecutor.cpp b/libdaggy/src/executors/task/NoopTaskExecutor.cpp similarity index 100% rename from daggy/src/executors/task/NoopTaskExecutor.cpp rename to libdaggy/src/executors/task/NoopTaskExecutor.cpp diff --git a/daggy/src/executors/task/SlurmTaskExecutor.cpp b/libdaggy/src/executors/task/SlurmTaskExecutor.cpp similarity index 100% rename from daggy/src/executors/task/SlurmTaskExecutor.cpp rename to libdaggy/src/executors/task/SlurmTaskExecutor.cpp diff --git a/daggy/src/loggers/CMakeLists.txt b/libdaggy/src/loggers/CMakeLists.txt similarity index 100% rename from daggy/src/loggers/CMakeLists.txt rename to libdaggy/src/loggers/CMakeLists.txt diff --git a/daggy/src/loggers/dag_run/CMakeLists.txt b/libdaggy/src/loggers/dag_run/CMakeLists.txt similarity index 100% rename from daggy/src/loggers/dag_run/CMakeLists.txt rename to libdaggy/src/loggers/dag_run/CMakeLists.txt diff --git a/daggy/src/loggers/dag_run/OStreamLogger.cpp b/libdaggy/src/loggers/dag_run/OStreamLogger.cpp similarity index 100% rename from daggy/src/loggers/dag_run/OStreamLogger.cpp rename to libdaggy/src/loggers/dag_run/OStreamLogger.cpp diff --git a/daggy/src/loggers/dag_run/RedisHelper.cpp b/libdaggy/src/loggers/dag_run/RedisHelper.cpp similarity index 100% rename from daggy/src/loggers/dag_run/RedisHelper.cpp rename to libdaggy/src/loggers/dag_run/RedisHelper.cpp diff --git a/daggy/src/loggers/dag_run/RedisLogger.cpp b/libdaggy/src/loggers/dag_run/RedisLogger.cpp similarity index 100% rename from daggy/src/loggers/dag_run/RedisLogger.cpp rename to libdaggy/src/loggers/dag_run/RedisLogger.cpp diff --git a/libdaggy/tests/CMakeLists.txt b/libdaggy/tests/CMakeLists.txt new file mode 100644 index 0000000..e4890c9 --- /dev/null +++ b/libdaggy/tests/CMakeLists.txt @@ -0,0 +1,20 @@ +project(libdaggy_tests) + +add_executable(${PROJECT_NAME} main.cpp + # unit tests + unit_dag.cpp + unit_dagrunner.cpp + unit_dagrun_loggers.cpp + unit_executor_forkingexecutor.cpp + unit_executor_slurmexecutor.cpp + unit_serialization.cpp + unit_threadpool.cpp + unit_utilities.cpp + # integration tests + int_basic.cpp + # Performance checks + perf_dag.cpp + ) +target_link_libraries(${PROJECT_NAME} libdaggy stdc++fs Catch2::Catch2) + +add_test(${PROJECT_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}) diff --git a/tests/int_basic.cpp b/libdaggy/tests/int_basic.cpp similarity index 100% rename from tests/int_basic.cpp rename to libdaggy/tests/int_basic.cpp diff --git a/libdaggy/tests/main.cpp b/libdaggy/tests/main.cpp new file mode 100644 index 0000000..4387fa6 --- /dev/null +++ b/libdaggy/tests/main.cpp @@ -0,0 +1,15 @@ +#include + +#include "daggy/DAG.hpp" + +#define CATCH_CONFIG_MAIN + +#include + +TEST_CASE("Sanity tests", "[sanity]") +{ + REQUIRE(1 == 1); +} + +// compile and run +// g++ -std=c++17 -o test test.cpp && ./test diff --git a/tests/perf_dag.cpp b/libdaggy/tests/perf_dag.cpp similarity index 100% rename from tests/perf_dag.cpp rename to libdaggy/tests/perf_dag.cpp diff --git a/tests/unit_dag.cpp b/libdaggy/tests/unit_dag.cpp similarity index 100% rename from tests/unit_dag.cpp rename to libdaggy/tests/unit_dag.cpp diff --git a/tests/unit_dagrun_loggers.cpp b/libdaggy/tests/unit_dagrun_loggers.cpp similarity index 100% rename from tests/unit_dagrun_loggers.cpp rename to libdaggy/tests/unit_dagrun_loggers.cpp diff --git a/tests/unit_dagrunner.cpp b/libdaggy/tests/unit_dagrunner.cpp similarity index 100% rename from tests/unit_dagrunner.cpp rename to libdaggy/tests/unit_dagrunner.cpp diff --git a/tests/unit_executor_forkingexecutor.cpp b/libdaggy/tests/unit_executor_forkingexecutor.cpp similarity index 100% rename from tests/unit_executor_forkingexecutor.cpp rename to libdaggy/tests/unit_executor_forkingexecutor.cpp diff --git a/tests/unit_executor_slurmexecutor.cpp b/libdaggy/tests/unit_executor_slurmexecutor.cpp similarity index 100% rename from tests/unit_executor_slurmexecutor.cpp rename to libdaggy/tests/unit_executor_slurmexecutor.cpp diff --git a/tests/unit_serialization.cpp b/libdaggy/tests/unit_serialization.cpp similarity index 100% rename from tests/unit_serialization.cpp rename to libdaggy/tests/unit_serialization.cpp diff --git a/tests/unit_server.cpp b/libdaggy/tests/unit_server.cpp similarity index 100% rename from tests/unit_server.cpp rename to libdaggy/tests/unit_server.cpp diff --git a/tests/unit_threadpool.cpp b/libdaggy/tests/unit_threadpool.cpp similarity index 100% rename from tests/unit_threadpool.cpp rename to libdaggy/tests/unit_threadpool.cpp diff --git a/tests/unit_utilities.cpp b/libdaggy/tests/unit_utilities.cpp similarity index 100% rename from tests/unit_utilities.cpp rename to libdaggy/tests/unit_utilities.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 3d54e2a..0000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -add_executable(unit_tests main.cpp - # unit tests - unit_dag.cpp - unit_dagrunner.cpp - unit_dagrun_loggers.cpp - unit_executor_forkingexecutor.cpp - unit_executor_slurmexecutor.cpp - unit_serialization.cpp - unit_server.cpp - unit_threadpool.cpp - unit_utilities.cpp - # integration tests - int_basic.cpp - # Performance checks - perf_dag.cpp - ) -target_link_libraries(unit_tests libdaggy stdc++fs Catch2::Catch2 curl) - -add_executable(integration_tests main.cpp - # unit tests - int_basic.cpp - ) -target_link_libraries(integration_tests libdaggy stdc++fs Catch2::Catch2 curl) - -add_executable(perf_tests main.cpp - # Performance checks - perf_dag.cpp - ) -target_link_libraries(perf_tests libdaggy stdc++fs Catch2::Catch2 curl) - -add_test(UNIT_TESTS ${CMAKE_BINARY_DIR}/tests/unit_tests) -add_test(INT_TESTS ${CMAKE_BINARY_DIR}/tests/integration_tests) -add_test(PERF_TESTS ${CMAKE_BINARY_DIR}/tests/perf_tests) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 6586e67..4fbb343 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,2 +1 @@ -add_subdirectory(daggyd) add_subdirectory(daggyc) diff --git a/utils/daggyd/CMakeLists.txt b/utils/daggyd/CMakeLists.txt deleted file mode 100644 index d5e83cc..0000000 --- a/utils/daggyd/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -project(daggyd) -file(GLOB SOURCES *.cpp) -add_executable(${PROJECT_NAME} ${SOURCES}) -target_link_libraries(${PROJECT_NAME} pistache stdc++fs rapidjson argparse libdaggy) \ No newline at end of file