Enhancing the performance of dag building by delaying validation until the end

tests/unit_utilities.cpp

@@ -5,6 +5,7 @@
 #include <iomanip>
 #include <algorithm>
 #include <iostream>
+#include <random>
 
 #include <catch2/catch.hpp>
 
@@ -180,3 +181,52 @@ TEST_CASE("dag_runner", "[utilities_dag_runner]") {
         REQUIRE(record.tasks["C_0"].children.empty());
     }
 }
+
+TEST_CASE("dag_runner_stress", "[utilities_dag_runner_stress]") {
+    daggy::executors::task::ForkingTaskExecutor ex(10);
+    std::stringstream ss;
+    daggy::loggers::dag_run::OStreamLogger logger(ss);
+
+
+    SECTION("Stress-test") {
+        static std::random_device dev;
+        static std::mt19937 rng(dev());
+        std::uniform_int_distribution<size_t> nDepDist(0, 10);
+
+        const size_t N_NODES = 100;
+        daggy::TaskSet tasks;
+        std::vector<fs::path> fileNames;
+        std::vector<std::string> taskNames;
+
+        for (size_t i = 0; i < N_NODES; ++i) {
+            std::string taskName = std::to_string(i);
+            std::uniform_int_distribution<size_t> depDist(i+1, N_NODES-1);
+            std::unordered_set<std::string> deps;
+            size_t nChildren = nDepDist(rng);
+            for (size_t c = 0; c < nChildren; ++c) {
+                deps.insert(std::to_string(depDist(rng)));
+            }
+            tasks.emplace(taskName, daggy::Task{
+                    .definedName = taskName,
+                    .job = { { "command", std::vector<std::string>{"/usr/bin/echo", taskName}}},
+                    .children = deps
+                    });
+        }
+
+        auto dag = daggy::buildDAGFromTasks(tasks);
+
+        auto runID = logger.startDAGRun("test_run", tasks);
+
+        auto tryDAG = daggy::runDAG(runID, ex, logger, dag);
+
+        REQUIRE(tryDAG.allVisited());
+
+        // Get the DAG Run Attempts
+        auto record = logger.getDAGRun(runID);
+        for (const auto & [k, attempts] : record.taskAttempts) {
+            REQUIRE(attempts.size() == 1);
+        }
+    }
+
+
+}