Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding a general logger and integrating it with daggyr

Browse Source
This commit is contained in:
Ian Roddis
2022-02-04 11:58:45 -04:00
parent 57e93b5045
commit ea3f67f226
8 changed files with 245 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
daggyr/libdaggyr/include/daggyr/Server.hpp
View File

@@ -5,6 +5,7 @@
#include <pistache/http.h>
#include <daggy/DAGRunner.hpp>
#include <daggy/GeneralLogger.hpp>
#include <daggy/ThreadPool.hpp>
#include <daggy/executors/task/DaggyRunnerTaskExecutor.hpp>
#include <daggy/executors/task/ForkingTaskExecutor.hpp>
@@ -25,7 +26,7 @@ namespace daggy::daggyr {
{
public:
Server(const Pistache::Address &listenSpec, ssize_t maxCores,
ssize_t maxMemoryMB);
ssize_t maxMemoryMB, GeneralLogger &logger);
~Server();
Server &setSSLCertificates(const fs::path &cert, const fs::path &key);
@@ -56,6 +57,7 @@ namespace daggy::daggyr {
Pistache::Rest::Description desc_;
Pistache::Rest::Router router_;
GeneralLogger &logger_;
executors::task::ForkingTaskExecutor executor_;
using TaskID = std::pair<DAGRunID, std::string>;

40
daggyr/libdaggyr/src/Server.cpp
View File

@@ -32,9 +32,10 @@ namespace daggy::daggyr {
}
Server::Server(const Pistache::Address &listenSpec, ssize_t maxCores,
ssize_t maxMemoryMB)
ssize_t maxMemoryMB, GeneralLogger &logger)
: endpoint_(listenSpec)
, desc_("Daggy Runner API", "0.1")
, logger_(logger)
, executor_(maxCores)
, maxCapacity_{maxCores, maxMemoryMB}
{
@@ -132,16 +133,21 @@ namespace daggy::daggyr {
if (!handleAuth(request))
return;
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto requestID = std::to_string(runID) + "/" + taskName;
logger_.info("Received task " + requestID);
Capacity resourcesUsed;
Task task;
logger_.debug([&] { return requestID + ": " + taskToJSON(task); });
try {
task = taskFromJSON(taskName, request.body());
resourcesUsed = capacityFromTask(task);
}
catch (std::exception &e) {
logger_.warn(
[&] { return requestID + ": Unable to parse task: " + e.what(); });
REQ_RESPONSE(Not_Acceptable, e.what());
}
@@ -153,6 +159,8 @@ namespace daggy::daggyr {
runningTasks_.emplace(std::move(tid), std::move(fut));
}
logger_.debug(requestID + ": Task successfully enqueued");
response.send(Pistache::Http::Code::Ok, "");
}
@@ -162,8 +170,9 @@ namespace daggy::daggyr {
if (!handleAuth(request))
return;
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto requestID = std::to_string(runID) + "/" + taskName;
auto taskID = std::make_pair(runID, taskName);
std::unordered_map<TaskID, daggy::executors::task::TaskFuture>::node_type
@@ -172,10 +181,16 @@ namespace daggy::daggyr {
{
std::lock_guard<std::mutex> lock(rtGuard_);
auto it = runningTasks_.find(taskID);
if (it == runningTasks_.end() || !it->second->ready()) {
if (it == runningTasks_.end()) {
logger_.warn(requestID + ": Polled about unknown task");
notFound = true;
}
else if (!it->second->ready()) {
logger_.debug(requestID + ": Polled but task not ready yet");
notFound = true;
}
else {
logger_.debug(requestID + ": Polled and ready.");
node = runningTasks_.extract(taskID);
}
}
@@ -189,9 +204,14 @@ namespace daggy::daggyr {
attemptRecordToJSON(node.mapped()->get()));
// If the promise fails, then reinsert the result for later polling
if (prom.isRejected()) {
logger_.warn(requestID +
": Record sent to poller, but failed to complete transfer.");
std::lock_guard<std::mutex> lock(rtGuard_);
runningTasks_.insert(std::move(node));
}
else {
logger_.debug(requestID + ": Record send successfully");
}
}
void Server::handleStopTask(const Pistache::Rest::Request &request,
@@ -200,9 +220,13 @@ namespace daggy::daggyr {
if (!handleAuth(request))
return;
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto runID = request.param(":runID").as<DAGRunID>();
auto taskName = request.param(":taskName").as<std::string>();
auto requestID = std::to_string(runID) + "/" + taskName;
logger_.debug([&] {
return std::to_string(runID) + "/" + taskName + ": Stop task received.";
});
executor_.stop(runID, taskName);
REQ_RESPONSE(Ok, "");