Kinesin/daggy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixing issue with resolving

Browse Source
This commit is contained in:
Ian Roddis
2021-12-24 10:44:19 -04:00
parent d832b935d7
commit 8ca5cdafe2
3 changed files with 66 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
daggyr/libdaggyr/src/Server.cpp
View File

@@ -204,9 +204,11 @@ namespace daggy::daggyr {
} }
std::cout << "Resolved " << it->runID << " / " << it->taskName std::cout << "Resolved " << it->runID << " / " << it->taskName
<< std::endl; << std::endl;
it = pending_.erase(it);
} }
else { else {
payload << R"("state": "PENDING")"; payload << R"("state": "PENDING")";
++it;
} }
payload << "}"; payload << "}";
} }

2
libdaggy/src/Utilities.cpp
View File

@@ -234,7 +234,7 @@ namespace daggy {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriter); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 2);
if (trace) { if (trace) {
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, http_trace); curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, http_trace);

23
libdaggy/src/executors/task/DaggyRunnerTaskExecutor.cpp
View File

@@ -149,7 +149,6 @@ std::future<AttemptRecord> DaggyRunnerTaskExecutor::execute(
double impact = double impact =
std::min(cores / caps.total.cores, memoryMB / caps.total.memoryMB); std::min(cores / caps.total.cores, memoryMB / caps.total.memoryMB);
std::cout << runner << ": " << impact << std::endl;
impacts.emplace_back(runner, impact); impacts.emplace_back(runner, impact);
} }
@@ -162,10 +161,8 @@ std::future<AttemptRecord> DaggyRunnerTaskExecutor::execute(
return fut; return fut;
} }
std::sort(impacts.begin(), impacts.end()); std::sort(impacts.begin(), impacts.end(),
for (const auto &[runner, impact] : impacts) { [](const auto &a, const auto &b) { return a.second < b.second; });
std::cout << "\t" << runner << ": " << impact << std::endl;
}
runner = impacts.back().first; runner = impacts.back().first;
auto &caps = runners_.at(runner); auto &caps = runners_.at(runner);
@@ -173,8 +170,6 @@ std::future<AttemptRecord> DaggyRunnerTaskExecutor::execute(
caps.current.memoryMB -= taskUsed.memoryMB; caps.current.memoryMB -= taskUsed.memoryMB;
} }
std::cout << "Queuing on runner: " << runner << std::endl;
std::stringstream ss; std::stringstream ss;
ss << runner << "/v1/task/" << runID << "/" << taskName; ss << runner << "/v1/task/" << runID << "/" << taskName;
auto url = ss.str(); auto url = ss.str();
@@ -221,7 +216,6 @@ void DaggyRunnerTaskExecutor::addRunner(const std::string &url)
void DaggyRunnerTaskExecutor::monitor() void DaggyRunnerTaskExecutor::monitor()
{ {
while (running_) { while (running_) {
{
std::unordered_map<std::pair<DAGRunID, std::string>, std::unordered_map<std::pair<DAGRunID, std::string>,
std::optional<AttemptRecord>> std::optional<AttemptRecord>>
resolvedJobs; resolvedJobs;
@@ -240,8 +234,7 @@ void DaggyRunnerTaskExecutor::monitor()
std::lock_guard<std::mutex> lock(runnersGuard_); std::lock_guard<std::mutex> lock(runnersGuard_);
for (auto &[runnerURL, caps] : runners_) { for (auto &[runnerURL, caps] : runners_) {
try { try {
const auto &[code, json] = const auto &[code, json] = JSON_HTTP_REQUEST(runnerURL + "/v1/poll");
JSON_HTTP_REQUEST(runnerURL + "/v1/poll");
if (code != HTTPCode::Ok) if (code != HTTPCode::Ok)
continue; continue;
@@ -249,8 +242,7 @@ void DaggyRunnerTaskExecutor::monitor()
for (size_t idx = 0; idx < tasks.Size(); ++idx) { for (size_t idx = 0; idx < tasks.Size(); ++idx) {
const auto &task = tasks[idx]; const auto &task = tasks[idx];
if (task["state"] == "PENDING") { if (task["state"] == "PENDING") {
resolvedJobs.emplace( resolvedJobs.emplace(std::make_pair(task["runID"].GetInt64(),
std::make_pair(task["runID"].GetInt64(),
task["taskName"].GetString()), task["taskName"].GetString()),
std::nullopt); std::nullopt);
} }
@@ -261,8 +253,7 @@ void DaggyRunnerTaskExecutor::monitor()
caps.current.cores += res.cores; caps.current.cores += res.cores;
caps.current.memoryMB += res.memoryMB; caps.current.memoryMB += res.memoryMB;
resolvedJobs.emplace(tid, resolvedJobs.emplace(tid, attemptRecordFromJSON(task["attempt"]));
attemptRecordFromJSON(task["attempt"]));
} }
} }
} }
@@ -283,8 +274,7 @@ void DaggyRunnerTaskExecutor::monitor()
if (task.retries == 0) { if (task.retries == 0) {
AttemptRecord record{ AttemptRecord record{
.rc = -1, .rc = -1, .executorLog = "Unable to query runner for progress"};
.executorLog = "Unable to query runner for progress"};
task.prom.set_value(std::move(record)); task.prom.set_value(std::move(record));
completedTasks.emplace_back(taskID); completedTasks.emplace_back(taskID);
} }
@@ -296,7 +286,6 @@ void DaggyRunnerTaskExecutor::monitor()
completedTasks.emplace_back(taskID); completedTasks.emplace_back(taskID);
} }
} }
}
for (const auto &tid : completedTasks) { for (const auto &tid : completedTasks) {
runningTasks_.extract(tid); runningTasks_.extract(tid);
} }