Refactoring RunState, fixing logic error in when to end errored DAG runs, added convenience functions to retry failed DAGs.

tests/unit_utilities.cpp

@@ -88,4 +88,49 @@ TEST_CASE("DAG Runner", "[utilities_dag_runner]") {
             REQUIRE(attempts.front().rc == 0);
         }
     }
+
+    SECTION("Recovery from Error") {
+      auto cleanup = []() {
+        // Cleanup
+        std::vector<fs::path> paths{"/tmp/rec_error_A", "/tmp/noexist" };
+        for (const auto & pth : paths) {
+          if (fs::exists(pth)) fs::remove_all(pth);
+        }
+      };
+
+      cleanup();
+
+
+        // daggy::loggers::dag_run::OStreamLogger logger(std::cout);
+
+        std::string goodPrefix = "/tmp/rec_error_";
+        std::string badPrefix = "/tmp/noexist/rec_error_";
+        std::string taskJSON = R"([{"name": "A", "command": ["/usr/bin/touch", ")"
+                               + goodPrefix + R"(A"], "children": ["C"]}, {"name": "B", "command": ["/usr/bin/touch", ")"
+                               + badPrefix + R"(B"], "children": ["C"]}, {"name": "C", "command": ["/usr/bin/touch", ")"
+                               + badPrefix + R"(C"]}])";
+        auto tasks = daggy::tasksFromJSON(taskJSON);
+        auto dag = daggy::buildDAGFromTasks(tasks);
+
+        auto runID = logger.startDAGRun("test_run", tasks);
+
+        auto tryDAG = daggy::runDAG(runID, tasks, ex, logger, dag);
+
+        REQUIRE(!tryDAG.allVisited());
+
+        // Create the missing dir, then continue to run the DAG
+        fs::create_directory("/tmp/noexist");
+        tryDAG.resetRunning();
+        auto endDAG = daggy::runDAG(runID, tasks, ex, logger, tryDAG);
+
+        REQUIRE(endDAG.allVisited());
+
+        // Get the DAG Run Attempts
+        auto record = logger.getDAGRun(runID);
+        REQUIRE(record.taskAttempts[0].size() == 1); // A ran fine
+        REQUIRE(record.taskAttempts[1].size() == 2); // B errored and had to be retried
+        REQUIRE(record.taskAttempts[2].size() == 1); // C wasn't run because B errored
+
+      cleanup();
+    }
 }