Adding Vue.js webui
Squashed commit of the following: commit 29571182b1ec3b5be2cec3212c2bea1121a3dac2 Author: Ian Roddis <tech@kinesin.ca> Date: Thu Feb 24 11:29:47 2022 -0400 Adding more elegant handling of tasks with no attempts commit 18c8ccb0863abbf6c9cc0efe5cc68df03a9eb80d Author: Ian Roddis <tech@kinesin.ca> Date: Thu Feb 24 11:18:59 2022 -0400 Better handling of no attempts at all commit 962f9f6e5e17f71bc3766553913774631f66e7ef Author: Ian Roddis <tech@kinesin.ca> Date: Thu Feb 24 11:10:28 2022 -0400 Adding fix for missing attempts commit 19b8203e952b3d21f4ff3f9b97a01c4d567ff1e7 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 16:56:37 2022 -0400 Adding webui instructions to readme commit 81383c80f01101828c0c49868916a2712d140f42 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 16:48:31 2022 -0400 Adding in route splatting to support static assets commit c9b39b307916c0fb1e88769d6986ddf7c3ba183a Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 12:11:11 2022 -0400 Cleanup commit 177819a1439cd1a0f32c652abf670f54457e105a Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 12:09:40 2022 -0400 Setting explicit url for extra CSS commit 78261129511c50657e7902934cee396eb1e4e3a8 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 12:08:27 2022 -0400 Moving webui commit 9f8db6e2c2c8a231060217cb82f1b13aabe4eae2 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 12:06:25 2022 -0400 Reorganizing elements, adding regex for run list commit f114250c9a506b2c0e9d642cc75749e99cc76cef Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 10:52:41 2022 -0400 Adding regex filtering to tasks commit 2de2f218416210443119aa88fa49c714197f4b16 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 10:42:22 2022 -0400 Adding in task details and getting the plumbing working commit 660a2078e22799ba51b4b8bbe5c12cd0f9315b0a Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 09:38:13 2022 -0400 Fixing remaining settings commit 1aa0dfe1c971a12dfed183586ee5a3206d452409 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 09:36:25 2022 -0400 Playing with settings commit 84cbd11c45651c7c6c96c16714e741b6aee10bc5 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 23 08:52:52 2022 -0400 Removing extra code commit 6e31646b7c62368cab22b3844a70943e0149ddc7 Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 22 17:29:47 2022 -0400 Adding linter, renaming components to meet standards, fixing some mixups in settings commit 225442ee5732d007867e485ccea05293e3e5e1b7 Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 22 17:25:27 2022 -0400 Fixing sorters commit eb0d7a4c4c30d8e8b43b574ed0c2f97515bb9353 Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 22 16:46:41 2022 -0400 Controls are coming together commit b1789d1cc3c0bae170e0ca1a47cccfd344197244 Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 22 11:08:09 2022 -0400 More refactoring commit 6d0afce429aad00864482a2cc7dd731a53312e14 Author: Ian Roddis <tech@kinesin.ca> Date: Sun Feb 20 22:29:43 2022 -0400 figuring out layout commit 6af498f3aa7fe2f45121df2278cdfac297165c5c Author: Ian Roddis <tech@kinesin.ca> Date: Sun Feb 20 12:30:49 2022 -0400 Migrating to prop drilling / emiting commit dffe7059ce01209d2def6ef7c03bc750e31fe741 Author: Ian Roddis <tech@kinesin.ca> Date: Fri Feb 18 17:20:46 2022 -0400 Checkpointing work for now commit d6428ad59c9c05ab7fba82ce3c0441ac3f568796 Author: Ian Roddis <tech@kinesin.ca> Date: Fri Feb 18 17:05:37 2022 -0400 Adding in toggling for states commit b9a4f2dc02f327d3529821e217d3b6a00a84f202 Author: Ian Roddis <tech@kinesin.ca> Date: Fri Feb 18 16:43:01 2022 -0400 Reorganizing everything commit d33691d022597d1ff8f588450e147c72555be9f4 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 17:04:54 2022 -0400 Removing console logging commit 4537376ccad6fc0c52f0a7cfd2b2bf23f708196c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 17:04:27 2022 -0400 Refresh timer working now commit 213a3da4fd07c82cd18cd8c3b2422ddc78bd6fb4 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 16:40:45 2022 -0400 Adding timer commit ff495ac69563689ff4fc07119936079e57608ea7 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 16:02:53 2022 -0400 Refactoring some code, adding in endpoint to kill a running task commit 97ff28b9b1910e03e0f2725a3f54d2a07e53714c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 14:56:15 2022 -0400 Renaming UI commit affab06ad657833b73588eac919250935b353f31 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 13:29:31 2022 -0400 moving to bootstrap commit c40a2e58a86362863c905470f4417753aaf0dac2 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 12:33:08 2022 -0400 adding task button commit 420463b8d7f964baa0dfc7c87c2e9024bc8284cc Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 10:51:11 2022 -0400 checkpoint commit a7aa3db731255e7e13bc58d901b8eb1e30ede39c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 09:33:01 2022 -0400 Fixing up state commit 361b4cbcd8f1268eb9b494084d6862a6ab8f3a27 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 09:29:14 2022 -0400 Fixing event callbacks commit 388cada692dc8d7e0eff611467d4c77ce897a54c Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 09:24:39 2022 -0400 Adding global state, task view and buttons commit cb5a3acef0bd982621678fbd44a133db56420871 Author: Ian Roddis <tech@kinesin.ca> Date: Wed Feb 16 07:49:30 2022 -0400 Adding RunView commit 4c78ef1250709e7c8f5ef3433640fd8d1d319a8d Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 15 17:20:23 2022 -0400 checkpoint commit 2c5b610101e9c18ef1ad8f962d7309b63c80743c Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 15 17:10:06 2022 -0400 Adding explicit payload headers, adding vue and react apps commit 95ac6c05903bc83c6934db58b48649eee2038c3d Author: Ian Roddis <tech@kinesin.ca> Date: Tue Feb 15 12:56:57 2022 -0400 Adding CORS support, rough-in of webui
This commit is contained in:
Ian Roddis
@@ -264,6 +264,7 @@ int main(int argc, char **argv)
|
||||
.default_value(false)
|
||||
.implicit_value(true);
|
||||
args.add_argument("-d", "--daemon").default_value(false).implicit_value(true);
|
||||
args.add_argument("--assets-dir").default_value(std::string{});
|
||||
args.add_argument("--config").default_value(std::string{});
|
||||
args.add_argument("--ip").default_value(std::string{"127.0.0.1"});
|
||||
args.add_argument("--port").default_value(2503u).action(
|
||||
@@ -281,6 +282,7 @@ int main(int argc, char **argv)
|
||||
bool verbose = args.get<bool>("--verbose");
|
||||
bool asDaemon = args.get<bool>("--daemon");
|
||||
auto configFile = args.get<std::string>("--config");
|
||||
auto staticAssetsDir = args.get<std::string>("--assets-dir");
|
||||
std::string listenIP = args.get<std::string>("--ip");
|
||||
auto listenPort = args.get<unsigned>("--port");
|
||||
size_t webThreads = 50;
|
||||
@@ -301,6 +303,8 @@ int main(int argc, char **argv)
|
||||
|
||||
if (doc.HasMember("ip"))
|
||||
listenIP = doc["ip"].GetString();
|
||||
if (doc.HasMember("assets-dir"))
|
||||
staticAssetsDir = doc["assets-dir"].GetString();
|
||||
if (doc.HasMember("port"))
|
||||
listenPort = doc["port"].GetInt();
|
||||
if (doc.HasMember("web-threads"))
|
||||
@@ -331,7 +335,8 @@ int main(int argc, char **argv)
|
||||
|
||||
Pistache::Address listenSpec(listenIP, listenPort);
|
||||
|
||||
daggy::daggyd::Server server(listenSpec, *logger, *executor, dagThreads);
|
||||
daggy::daggyd::Server server(listenSpec, *logger, *executor, dagThreads,
|
||||
staticAssetsDir);
|
||||
server.init(webThreads);
|
||||
server.start();
|
||||
|
||||
|
||||
@@ -22,7 +22,8 @@ namespace daggy::daggyd {
|
||||
public:
|
||||
Server(const Pistache::Address &listenSpec,
|
||||
loggers::dag_run::DAGRunLogger &logger,
|
||||
executors::task::TaskExecutor &executor, size_t nDAGRunners);
|
||||
executors::task::TaskExecutor &executor, size_t nDAGRunners,
|
||||
const fs::path &staticAssetsDir);
|
||||
~Server();
|
||||
|
||||
Server &setSSLCertificates(const fs::path &cert, const fs::path &key);
|
||||
@@ -40,7 +41,7 @@ namespace daggy::daggyd {
|
||||
void queueDAG_(DAGRunID runID, const TaskDAG &dag,
|
||||
const TaskParameters &taskParameters);
|
||||
|
||||
DAGGY_REST_HANDLER(handleRoot); // X
|
||||
DAGGY_REST_HANDLER(handleStatic); // X
|
||||
DAGGY_REST_HANDLER(handleReady); // X
|
||||
DAGGY_REST_HANDLER(handleQueryDAGs); // X
|
||||
DAGGY_REST_HANDLER(handleRunDAG); // X
|
||||
@@ -53,6 +54,7 @@ namespace daggy::daggyd {
|
||||
DAGGY_REST_HANDLER(handleStopTask); // X
|
||||
DAGGY_REST_HANDLER(handleGetTaskState); // X
|
||||
DAGGY_REST_HANDLER(handleSetTaskState); // X
|
||||
DAGGY_REST_HANDLER(handleCORS);
|
||||
|
||||
bool handleAuth(const Pistache::Rest::Request &request);
|
||||
|
||||
@@ -63,6 +65,7 @@ namespace daggy::daggyd {
|
||||
loggers::dag_run::DAGRunLogger &logger_;
|
||||
executors::task::TaskExecutor &executor_;
|
||||
ThreadPool runnerPool_;
|
||||
fs::path staticAssetsDir_;
|
||||
|
||||
std::mutex runnerGuard_;
|
||||
std::unordered_map<DAGRunID, std::shared_ptr<DAGRunner>> runners_;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <daggy/Serialization.hpp>
|
||||
#include <daggy/Utilities.hpp>
|
||||
#include <daggyd/Server.hpp>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
@@ -20,14 +21,20 @@ using namespace Pistache;
|
||||
|
||||
namespace daggy::daggyd {
|
||||
|
||||
bool requestIsForJSON(const Pistache::Rest::Request &request)
|
||||
void addResponseHeaders(Pistache::Http::ResponseWriter &response)
|
||||
{
|
||||
auto acceptedMimeTypes =
|
||||
request.headers().get<Pistache::Http::Header::Accept>()->media();
|
||||
auto fit =
|
||||
std::find(acceptedMimeTypes.begin(), acceptedMimeTypes.end(),
|
||||
Pistache::Http::Mime::MediaType::fromString("text/html"));
|
||||
return fit == acceptedMimeTypes.end();
|
||||
response.headers().add(
|
||||
std::make_shared<Pistache::Http::Header::AccessControlAllowOrigin>(
|
||||
"*"));
|
||||
response.headers().add(
|
||||
std::make_shared<Pistache::Http::Header::AccessControlAllowHeaders>(
|
||||
"content-type"));
|
||||
response.headers().add(
|
||||
std::make_shared<Pistache::Http::Header::AccessControlAllowMethods>(
|
||||
"PUT, OPTIONS, PATCH, GET, HEAD, CONNECT, DELETE, POST, TRACE"));
|
||||
response.headers().add(
|
||||
std::make_shared<Pistache::Http::Header::ContentType>(
|
||||
"application/javascript"));
|
||||
}
|
||||
|
||||
void Server::init(size_t threads)
|
||||
@@ -44,12 +51,14 @@ namespace daggy::daggyd {
|
||||
|
||||
Server::Server(const Pistache::Address &listenSpec,
|
||||
loggers::dag_run::DAGRunLogger &logger,
|
||||
executors::task::TaskExecutor &executor, size_t nDAGRunners)
|
||||
executors::task::TaskExecutor &executor, size_t nDAGRunners,
|
||||
const fs::path &staticAssetsDir)
|
||||
: endpoint_(listenSpec)
|
||||
, desc_("Daggy API", "0.1")
|
||||
, logger_(logger)
|
||||
, executor_(executor)
|
||||
, runnerPool_(nDAGRunners)
|
||||
, staticAssetsDir_(staticAssetsDir)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -91,6 +100,19 @@ namespace daggy::daggyd {
|
||||
desc_.response(Http::Code::Internal_Server_Error,
|
||||
R"({"error": "An error occurred with the backend"})");
|
||||
|
||||
desc_.route(desc_.get("/"))
|
||||
.bind(&Server::handleStatic, this)
|
||||
.response(Http::Code::Ok, "Serve static assets")
|
||||
.hide();
|
||||
desc_.route(desc_.get("/*"))
|
||||
.bind(&Server::handleStatic, this)
|
||||
.response(Http::Code::Ok, "Serve static assets")
|
||||
.hide();
|
||||
desc_.route(desc_.get("/*/*"))
|
||||
.bind(&Server::handleStatic, this)
|
||||
.response(Http::Code::Ok, "Serve static assets")
|
||||
.hide();
|
||||
|
||||
desc_.schemes(Rest::Scheme::Http)
|
||||
.basePath("/v1")
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -101,11 +123,6 @@ namespace daggy::daggyd {
|
||||
.response(Http::Code::Ok, "Response to the /ready call")
|
||||
.hide();
|
||||
|
||||
desc_.route(desc_.get("/"))
|
||||
.bind(&Server::handleRoot, this)
|
||||
.response(Http::Code::Ok, "Response to the /ready call")
|
||||
.hide();
|
||||
|
||||
auto versionPath = desc_.path("/v1");
|
||||
|
||||
/*
|
||||
@@ -113,6 +130,7 @@ namespace daggy::daggyd {
|
||||
*/
|
||||
auto dagRunsPath = versionPath.path("/dagruns");
|
||||
|
||||
dagRunsPath.route(desc_.options("/")).bind(&Server::handleCORS, this);
|
||||
dagRunsPath.route(desc_.get("/"))
|
||||
.bind(&Server::handleQueryDAGs, this)
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -123,11 +141,14 @@ namespace daggy::daggyd {
|
||||
*/
|
||||
auto dagRunPath = versionPath.path("/dagrun");
|
||||
|
||||
dagRunPath.route(desc_.options("/")).bind(&Server::handleCORS, this);
|
||||
dagRunPath.route(desc_.post("/"))
|
||||
.bind(&Server::handleRunDAG, this)
|
||||
.produces(MIME(Application, Json))
|
||||
.response(Http::Code::Ok, "Run a DAG");
|
||||
|
||||
dagRunPath.route(desc_.options("/validate"))
|
||||
.bind(&Server::handleCORS, this);
|
||||
dagRunPath.route(desc_.post("/validate"))
|
||||
.bind(&Server::handleValidateDAG, this)
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -138,6 +159,8 @@ namespace daggy::daggyd {
|
||||
*/
|
||||
auto specificDAGRunPath = dagRunPath.path("/:runID");
|
||||
|
||||
specificDAGRunPath.route(desc_.options("/"))
|
||||
.bind(&Server::handleCORS, this);
|
||||
specificDAGRunPath.route(desc_.del("/"))
|
||||
.bind(&Server::handleStopDAGRun, this)
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -148,12 +171,16 @@ namespace daggy::daggyd {
|
||||
.produces(MIME(Application, Json))
|
||||
.response(Http::Code::Ok, "Full DAG Run");
|
||||
|
||||
specificDAGRunPath.route(desc_.options("/state"))
|
||||
.bind(&Server::handleCORS, this);
|
||||
specificDAGRunPath.route(desc_.get("/state"))
|
||||
.bind(&Server::handleGetDAGRunState, this)
|
||||
.produces(MIME(Application, Json))
|
||||
.response(Http::Code::Ok,
|
||||
"Structure of a DAG and DAG and Task run states");
|
||||
|
||||
specificDAGRunPath.route(desc_.options("/state/:state"))
|
||||
.bind(&Server::handleCORS, this);
|
||||
specificDAGRunPath.route(desc_.patch("/state/:state"))
|
||||
.bind(&Server::handleSetDAGRunState, this)
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -163,6 +190,7 @@ namespace daggy::daggyd {
|
||||
Task paths
|
||||
*/
|
||||
auto taskPath = specificDAGRunPath.path("/task/:taskName");
|
||||
taskPath.route(desc_.options("/")).bind(&Server::handleCORS, this);
|
||||
taskPath.route(desc_.get("/"))
|
||||
.bind(&Server::handleGetTask, this)
|
||||
.produces(MIME(Application, Json))
|
||||
@@ -177,22 +205,33 @@ namespace daggy::daggyd {
|
||||
*/
|
||||
auto taskStatePath = taskPath.path("/state");
|
||||
|
||||
taskStatePath.route(desc_.options("/")).bind(&Server::handleCORS, this);
|
||||
taskStatePath.route(desc_.get("/"))
|
||||
.bind(&Server::handleGetTaskState, this)
|
||||
.produces(MIME(Application, Json))
|
||||
.response(Http::Code::Ok, "Get a task state");
|
||||
|
||||
taskStatePath.route(desc_.options("/:state"))
|
||||
.bind(&Server::handleCORS, this);
|
||||
taskStatePath.route(desc_.patch("/:state"))
|
||||
.bind(&Server::handleSetTaskState, this)
|
||||
.produces(MIME(Application, Json))
|
||||
.response(Http::Code::Ok, "Set a task state");
|
||||
}
|
||||
|
||||
void Server::handleCORS(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
response.send(Pistache::Http::Code::Ok, "");
|
||||
}
|
||||
|
||||
void Server::handleRunDAG(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
addResponseHeaders(response);
|
||||
|
||||
DAGRunID runID = 0;
|
||||
try {
|
||||
@@ -216,6 +255,7 @@ namespace daggy::daggyd {
|
||||
void Server::handleValidateDAG(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
try {
|
||||
dagFromJSON(request.body());
|
||||
response.send(Pistache::Http::Code::Ok, R"({"valid": true}\n)");
|
||||
@@ -231,6 +271,7 @@ namespace daggy::daggyd {
|
||||
void Server::handleQueryDAGs(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
@@ -241,116 +282,63 @@ namespace daggy::daggyd {
|
||||
tag = request.query().get("tag").value();
|
||||
}
|
||||
|
||||
bool isJSON = requestIsForJSON(request);
|
||||
|
||||
if (request.hasParam(":all")) {
|
||||
auto val = request.query().get(":all").value();
|
||||
if (request.query().has("all")) {
|
||||
auto val = request.query().get("all").value();
|
||||
if (val == "true" or val == "1") {
|
||||
all = true;
|
||||
}
|
||||
}
|
||||
else if (!isJSON) {
|
||||
all = true;
|
||||
}
|
||||
|
||||
auto dagRuns = logger_.queryDAGRuns(tag, all);
|
||||
std::stringstream ss;
|
||||
if (isJSON) {
|
||||
// default to json
|
||||
ss << '[';
|
||||
// default to json
|
||||
ss << '[';
|
||||
|
||||
bool first = true;
|
||||
for (const auto &run : dagRuns) {
|
||||
if (first) {
|
||||
first = false;
|
||||
bool first = true;
|
||||
for (const auto &run : dagRuns) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
else {
|
||||
ss << ", ";
|
||||
}
|
||||
|
||||
ss << " {"
|
||||
<< R"("runID": )" << run.runID << ',' << R"("tag": )"
|
||||
<< std::quoted(run.tag) << ","
|
||||
<< R"("state": )" << std::quoted(run.runState._to_string()) << ","
|
||||
<< R"("startTime": )" << std::quoted(timePointToString(run.startTime))
|
||||
<< ',' << R"("lastUpdate": )"
|
||||
<< std::quoted(timePointToString(run.lastUpdate)) << ','
|
||||
<< R"("taskCounts": {)";
|
||||
bool firstState = true;
|
||||
for (const auto &[state, count] : run.taskStateCounts) {
|
||||
if (firstState) {
|
||||
firstState = false;
|
||||
}
|
||||
else {
|
||||
ss << ", ";
|
||||
}
|
||||
|
||||
ss << " {"
|
||||
<< R"("runID": )" << run.runID << ',' << R"("tag": )"
|
||||
<< std::quoted(run.tag) << ","
|
||||
<< R"("startTime": )"
|
||||
<< std::quoted(timePointToString(run.startTime)) << ','
|
||||
<< R"("lastUpdate": )"
|
||||
<< std::quoted(timePointToString(run.lastUpdate)) << ','
|
||||
<< R"("taskCounts": {)";
|
||||
bool firstState = true;
|
||||
for (const auto &[state, count] : run.taskStateCounts) {
|
||||
if (firstState) {
|
||||
firstState = false;
|
||||
}
|
||||
else {
|
||||
ss << ", ";
|
||||
}
|
||||
ss << std::quoted(state._to_string()) << ':' << count;
|
||||
}
|
||||
ss << '}' // end of taskCounts
|
||||
<< '}'; // end of item
|
||||
ss << std::quoted(state._to_string()) << ':' << count;
|
||||
}
|
||||
ss << "]\n";
|
||||
}
|
||||
else {
|
||||
// HTML
|
||||
ss << "<html><head><title>Daggy "
|
||||
"Runs</title><meta http-equiv=\"refresh\" "
|
||||
"content=\"10\"></head><body><center><h2>Current Runs</h2><br>";
|
||||
if (!dagRuns.empty()) {
|
||||
std::sort(dagRuns.begin(), dagRuns.end(),
|
||||
[](const auto &a, const auto &b) {
|
||||
return a.startTime > b.startTime;
|
||||
});
|
||||
ss << "<table><tr><th>Run ID</th><th>Tag</th><th>State</th><th># "
|
||||
"Tasks</th><th>Start Time</th><th>Last "
|
||||
"Update</th><th>Queued</th><th>Running</th><th>Retry</"
|
||||
"th><th>Errored</th><th>Completed</th></tr>";
|
||||
for (auto &ds : dagRuns) {
|
||||
size_t nTasks = 0;
|
||||
for (const auto &[k, cnt] : ds.taskStateCounts)
|
||||
nTasks += cnt;
|
||||
|
||||
auto stateURL = [&](RunState state) {
|
||||
std::stringstream ss;
|
||||
ss << R"(<a href="/v1/dagrun/)" << ds.runID
|
||||
<< "/?state=" << state._to_string() << "\">"
|
||||
<< ds.taskStateCounts[state] << "</a>";
|
||||
return ss.str();
|
||||
};
|
||||
|
||||
ss << "<tr>"
|
||||
<< R"(<td><a href="/v1/dagrun/)" << ds.runID << R"(">)" << ds.runID
|
||||
<< "</a></td>"
|
||||
<< "<td>" << ds.tag << "</td>"
|
||||
<< "<td>" << ds.runState << "</td>"
|
||||
<< "<td>" << nTasks << "</td>"
|
||||
<< "<td>" << timePointToString(ds.startTime) << "</td>"
|
||||
<< "<td>" << timePointToString(ds.lastUpdate) << "</td>"
|
||||
<< "<td>" << stateURL(RunState::QUEUED) << "</td>"
|
||||
<< "<td>" << stateURL(RunState::RUNNING) << "</td>"
|
||||
<< "<td>" << stateURL(RunState::RETRY) << "</td>"
|
||||
<< "<td>" << stateURL(RunState::ERRORED) << "</td>"
|
||||
<< "<td>" << stateURL(RunState::COMPLETED) << "</td>"
|
||||
<< "</tr>";
|
||||
}
|
||||
ss << "</table>";
|
||||
}
|
||||
ss << "</body></html>\n";
|
||||
ss << '}' // end of taskCounts
|
||||
<< '}'; // end of item
|
||||
}
|
||||
ss << "]\n";
|
||||
response.send(Pistache::Http::Code::Ok, ss.str());
|
||||
}
|
||||
|
||||
void Server::handleGetDAGRun(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
if (!request.hasParam(":runID")) {
|
||||
REQ_RESPONSE(Not_Found, "No runID provided in URL");
|
||||
}
|
||||
auto runID = request.param(":runID").as<size_t>();
|
||||
auto run = logger_.getDAGRun(runID);
|
||||
bool isJSON = requestIsForJSON(request);
|
||||
auto runID = request.param(":runID").as<size_t>();
|
||||
auto run = logger_.getDAGRun(runID);
|
||||
|
||||
std::optional<RunState> filterState;
|
||||
if (request.query().has("state")) {
|
||||
@@ -359,136 +347,72 @@ namespace daggy::daggyd {
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
if (isJSON) {
|
||||
bool first = true;
|
||||
ss << "{"
|
||||
<< R"("runID": )" << runID << ',' << R"("tag": )"
|
||||
<< std::quoted(run.dagSpec.tag) << ',' << R"("tasks": )"
|
||||
<< tasksToJSON(run.dagSpec.tasks) << ',';
|
||||
bool first = true;
|
||||
ss << "{"
|
||||
<< R"("runID": )" << runID << ',' << R"("tag": )"
|
||||
<< std::quoted(run.dagSpec.tag) << ',' << R"("tasks": )"
|
||||
<< tasksToJSON(run.dagSpec.tasks) << ',';
|
||||
|
||||
// task run states
|
||||
ss << R"("taskStates": { )";
|
||||
first = true;
|
||||
for (const auto &[name, state] : run.taskRunStates) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << std::quoted(name) << ": " << std::quoted(state._to_string());
|
||||
// task run states
|
||||
ss << R"("taskStates": { )";
|
||||
first = true;
|
||||
for (const auto &[name, state] : run.taskRunStates) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
ss << "},";
|
||||
|
||||
// Attempt records
|
||||
first = true;
|
||||
ss << R"("taskAttempts": { )";
|
||||
for (const auto &[taskName, attempts] : run.taskAttempts) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << std::quoted(taskName) << ": [";
|
||||
bool firstAttempt = true;
|
||||
for (const auto &attempt : attempts) {
|
||||
if (firstAttempt) {
|
||||
firstAttempt = false;
|
||||
}
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << attemptRecordToJSON(attempt);
|
||||
}
|
||||
ss << ']';
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << "},";
|
||||
|
||||
// DAG state changes
|
||||
first = true;
|
||||
ss << R"("dagStateChanges": [ )";
|
||||
for (const auto &change : run.dagStateChanges) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << stateUpdateRecordToJSON(change);
|
||||
}
|
||||
ss << "]";
|
||||
ss << "}\n";
|
||||
ss << std::quoted(name) << ": " << std::quoted(state._to_string());
|
||||
}
|
||||
else {
|
||||
std::unordered_map<RunState, size_t> stateCounts;
|
||||
for (const auto &[_, state] : run.taskRunStates) {
|
||||
stateCounts[state]++;
|
||||
ss << "},";
|
||||
|
||||
// Attempt records
|
||||
first = true;
|
||||
ss << R"("taskAttempts": { )";
|
||||
for (const auto &[taskName, attempts] : run.taskAttempts) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
|
||||
ss << R"(<html>
|
||||
<head>
|
||||
<title>Details for RunID )"
|
||||
<< runID << R"(</title>
|
||||
<meta http-equiv="refresh" "content="10">
|
||||
<script>
|
||||
function resubmit(run_id, task_name) {
|
||||
let url = "/v1/dagrun/" + run_id + "/task/" + task_name + "/state/QUEUED";
|
||||
fetch(url, { method: 'PATCH' })
|
||||
.then(res => console.log('Resubmitted task'))
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<center>
|
||||
<h2>Summary</h2>
|
||||
<table><tr><th>Run ID</th><th>Tag</th><th>State</th>
|
||||
<th>#Tasks</th>
|
||||
<th>Queued</th><th>Running</th><th>Retry</th>
|
||||
<th>Errored</th><th>Completed</th></tr>
|
||||
<tr>)"
|
||||
<< "<td>" << runID << "</td>"
|
||||
<< "<td>" << run.dagSpec.tag << "</td>"
|
||||
<< "<td>" << run.dagStateChanges.back().state << "</td>"
|
||||
<< "<td>" << run.dagSpec.tasks.size() << "</td>"
|
||||
<< "<td>" << stateCounts[RunState::QUEUED] << "</td>"
|
||||
<< "<td>" << stateCounts[RunState::RUNNING] << "</td>"
|
||||
<< "<td>" << stateCounts[RunState::RETRY] << "</td>"
|
||||
<< "<td>" << stateCounts[RunState::ERRORED] << "</td>"
|
||||
<< "<td>" << stateCounts[RunState::COMPLETED] << "</td>"
|
||||
<< "</tr></table>"
|
||||
<< "<h2>Task Details</h2>"
|
||||
<< "<table><tr><th>Task Name</th><th> State</th><th>Last "
|
||||
"Update</th><th> Logs</th></tr>";
|
||||
|
||||
for (const auto &[taskName, task] : run.dagSpec.tasks) {
|
||||
auto taskState = run.taskRunStates.at(taskName);
|
||||
if (filterState and taskState != *filterState)
|
||||
continue;
|
||||
std::string retryButton = "";
|
||||
if (taskState == +RunState::ERRORED) {
|
||||
retryButton = " <a href=\"#\" onclick=\"resubmit(" +
|
||||
std::to_string(runID) + ", '" + taskName +
|
||||
"');\">Retry</a>";
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << std::quoted(taskName) << ": [";
|
||||
bool firstAttempt = true;
|
||||
for (const auto &attempt : attempts) {
|
||||
if (firstAttempt) {
|
||||
firstAttempt = false;
|
||||
}
|
||||
ss << "<tr>"
|
||||
<< "<td>" << taskName << "</td>"
|
||||
<< "<td>" << run.taskRunStates.at(taskName) << "</td>"
|
||||
<< "<td>"
|
||||
<< timePointToString(run.taskStateChanges.at(taskName).back().time)
|
||||
<< "</td>"
|
||||
<< "<td><a href=\"/v1/dagrun/" << runID << "/task/" << taskName
|
||||
<< "\">Logs</a>" << retryButton << "</td>"
|
||||
<< "</tr>";
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << attemptRecordToJSON(attempt);
|
||||
}
|
||||
ss << "</table></center></body></html>";
|
||||
ss << ']';
|
||||
}
|
||||
ss << "},";
|
||||
|
||||
// DAG state changes
|
||||
first = true;
|
||||
ss << R"("dagStateChanges": [ )";
|
||||
for (const auto &change : run.dagStateChanges) {
|
||||
if (first) {
|
||||
first = false;
|
||||
}
|
||||
else {
|
||||
ss << ',';
|
||||
}
|
||||
ss << stateUpdateRecordToJSON(change);
|
||||
}
|
||||
ss << "]";
|
||||
ss << "}\n";
|
||||
response.send(Pistache::Http::Code::Ok, ss.str());
|
||||
}
|
||||
|
||||
void Server::handleStopDAGRun(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
if (!request.hasParam(":runID")) {
|
||||
@@ -556,6 +480,7 @@ namespace daggy::daggyd {
|
||||
{
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
addResponseHeaders(response);
|
||||
|
||||
// TODO handle state transition
|
||||
DAGRunID runID = request.param(":runID").as<DAGRunID>();
|
||||
@@ -609,63 +534,54 @@ namespace daggy::daggyd {
|
||||
void Server::handleGetTask(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
auto runID = request.param(":runID").as<DAGRunID>();
|
||||
auto taskName = request.param(":taskName").as<std::string>();
|
||||
bool isJSON = requestIsForJSON(request);
|
||||
|
||||
std::stringstream ss;
|
||||
if (isJSON) {
|
||||
Task task;
|
||||
try {
|
||||
task = logger_.getTask(runID, taskName);
|
||||
}
|
||||
catch (std::exception &e) {
|
||||
REQ_RESPONSE(Not_Found, e.what());
|
||||
}
|
||||
ss << taskToJSON(task);
|
||||
Task task;
|
||||
try {
|
||||
task = logger_.getTask(runID, taskName);
|
||||
}
|
||||
else {
|
||||
std::optional<loggers::dag_run::TaskRecord> tr;
|
||||
try {
|
||||
tr.emplace(logger_.getTaskRecord(runID, taskName));
|
||||
}
|
||||
catch (std::exception &e) {
|
||||
REQ_RESPONSE(Not_Found, e.what());
|
||||
}
|
||||
ss << "<html><title>Task Details for " << runID << " / " << taskName
|
||||
<< "</title><body>"
|
||||
<< "<table>"
|
||||
<< "<tr><th>Name</th><td>" << taskName << "</td></tr>"
|
||||
<< "<tr><th>State</th><td>" << tr->state << "</td></tr>"
|
||||
<< "<tr><th>Definition</th><td>" << taskToJSON(tr->task)
|
||||
<< "</td></tr>"
|
||||
<< "<tr><th colspan=2>Attempts</th></tr>";
|
||||
|
||||
std::sort(tr->attempts.begin(), tr->attempts.end(),
|
||||
[](const auto &a, const auto &b) {
|
||||
return a.startTime < b.startTime;
|
||||
});
|
||||
|
||||
for (const auto &attempt : tr->attempts) {
|
||||
ss << "<tr><td valign=top>" << timePointToString(attempt.startTime)
|
||||
<< "</td><td><pre>rc: " << attempt.rc
|
||||
<< "\n\nstdout:\n--------------\n"
|
||||
<< attempt.outputLog << "\n\nstderr:\n--------------\n"
|
||||
<< attempt.errorLog << "\n\nexecutor:\n--------------\n"
|
||||
<< attempt.executorLog << "</pre></td></tr>";
|
||||
}
|
||||
|
||||
ss << "</table></body></html>\n";
|
||||
catch (std::exception &e) {
|
||||
REQ_RESPONSE(Not_Found, e.what());
|
||||
}
|
||||
ss << taskToJSON(task);
|
||||
response.send(Pistache::Http::Code::Ok, ss.str());
|
||||
}
|
||||
|
||||
void Server::handleStopTask(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
auto runID = request.param(":runID").as<DAGRunID>();
|
||||
auto taskName = request.param(":taskName").as<std::string>();
|
||||
|
||||
std::shared_ptr<DAGRunner> runner{nullptr};
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(runnerGuard_);
|
||||
auto it = runners_.find(runID);
|
||||
if (runners_.find(runID) != runners_.end()) {
|
||||
runner = it->second;
|
||||
}
|
||||
}
|
||||
|
||||
if (runner) {
|
||||
runner->stopTask(taskName);
|
||||
}
|
||||
REQ_RESPONSE(Ok, "");
|
||||
}
|
||||
|
||||
void Server::handleGetTaskState(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
@@ -685,28 +601,10 @@ namespace daggy::daggyd {
|
||||
}
|
||||
}
|
||||
|
||||
void Server::handleStopTask(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
auto runID = request.param(":runID").as<DAGRunID>();
|
||||
auto taskName = request.param(":taskName").as<std::string>();
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(runnerGuard_);
|
||||
auto it = runners_.find(runID);
|
||||
if (runners_.find(runID) != runners_.end()) {
|
||||
it->second->stopTask(taskName);
|
||||
}
|
||||
}
|
||||
response.send(Pistache::Http::Code::Ok, "");
|
||||
}
|
||||
|
||||
void Server::handleSetTaskState(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
if (!handleAuth(request))
|
||||
return;
|
||||
|
||||
@@ -728,16 +626,59 @@ namespace daggy::daggyd {
|
||||
void Server::handleReady(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
response.send(Pistache::Http::Code::Ok, R"({ "msg": "Ya like DAGs?"}\n)");
|
||||
addResponseHeaders(response);
|
||||
REQ_RESPONSE(Ok, "Ya like DAGs?");
|
||||
}
|
||||
|
||||
void Server::handleRoot(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
void Server::handleStatic(const Pistache::Rest::Request &request,
|
||||
Pistache::Http::ResponseWriter response)
|
||||
{
|
||||
addResponseHeaders(response);
|
||||
std::string file = "index.html";
|
||||
|
||||
auto splats = request.splat();
|
||||
if (!splats.empty()) {
|
||||
file = splats[0].as<std::string>();
|
||||
for (size_t i = 1; i < splats.size(); ++i)
|
||||
file += "/" + splats[i].as<std::string>();
|
||||
}
|
||||
|
||||
auto fn = staticAssetsDir_ / file;
|
||||
|
||||
auto ext = fn.extension();
|
||||
|
||||
if (!fs::exists(fn)) {
|
||||
std::cout << "Can't find " << fn << std::endl;
|
||||
REQ_RESPONSE(Not_Found, "");
|
||||
}
|
||||
|
||||
std::string contentType;
|
||||
if (ext == ".svg") {
|
||||
contentType = "image/svg+xml";
|
||||
}
|
||||
else if (ext == ".html") {
|
||||
contentType = "text/html";
|
||||
}
|
||||
else if (ext == ".css") {
|
||||
contentType = "text/css";
|
||||
}
|
||||
else if (ext == ".js") {
|
||||
contentType = "text/javascript";
|
||||
}
|
||||
else {
|
||||
REQ_RESPONSE(Bad_Request, "I don't know how to serve that kind of file");
|
||||
}
|
||||
|
||||
response.headers().remove<Pistache::Http::Header::ContentType>();
|
||||
response.headers().add(
|
||||
std::make_shared<Pistache::Http::Header::Location>("/v1/dagruns"));
|
||||
response.send(Pistache::Http::Code::Moved_Permanently,
|
||||
R"({ "msg": "These are the dags you are looking for"}\n)");
|
||||
std::make_shared<Pistache::Http::Header::ContentType>(contentType));
|
||||
|
||||
std::stringstream ss;
|
||||
std::ifstream ifh;
|
||||
ifh.open(fn, std::ios::binary);
|
||||
ss << ifh.rdbuf();
|
||||
|
||||
response.send(Pistache::Http::Code::Ok, ss.str());
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -26,7 +26,8 @@ TEST_CASE("rest_endpoint", "[server_basic]")
|
||||
|
||||
const size_t nDAGRunners = 10, nWebThreads = 10;
|
||||
|
||||
daggy::daggyd::Server server(listenSpec, logger, executor, nDAGRunners);
|
||||
daggy::daggyd::Server server(listenSpec, logger, executor, nDAGRunners,
|
||||
"/dev/null");
|
||||
server.init(nWebThreads);
|
||||
server.start();
|
||||
|
||||
@@ -165,12 +166,13 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
|
||||
{
|
||||
std::stringstream ss;
|
||||
daggy::executors::task::ForkingTaskExecutor executor(10);
|
||||
daggy::loggers::dag_run::OStreamLogger logger(ss);
|
||||
daggy::loggers::dag_run::OStreamLogger logger(std::cout);
|
||||
Pistache::Address listenSpec("localhost", Pistache::Port(0));
|
||||
|
||||
const size_t nDAGRunners = 10, nWebThreads = 10;
|
||||
|
||||
daggy::daggyd::Server server(listenSpec, logger, executor, nDAGRunners);
|
||||
daggy::daggyd::Server server(listenSpec, logger, executor, nDAGRunners,
|
||||
"/dev/null");
|
||||
server.init(nWebThreads);
|
||||
server.start();
|
||||
|
||||
@@ -249,7 +251,7 @@ TEST_CASE("Server cancels and resumes execution", "[server_resume]")
|
||||
"PATCH");
|
||||
|
||||
// Wait for run to complete
|
||||
std::this_thread::sleep_for(5s);
|
||||
std::this_thread::sleep_for(3s);
|
||||
REQUIRE(logger.getDAGRunState(runID) == +daggy::RunState::COMPLETED);
|
||||
|
||||
REQUIRE(fs::exists("resume_touch_c"));
|
||||
|
||||
Reference in New Issue
Block a user