From 21fcfa7758b83216fa5a823c3ed0eb6a66bdf0a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Tue, 20 Dec 2022 14:47:32 +0100
Subject: [PATCH] use_dll: fixes to parallel compilation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
— No longer call std::exit() from threads when compilation fails, that function
is marked as not thread-safe under GNU/Linux; and it leads to deadlocks under
Windows. Rather store the list of failed objects, and exit with a message and
an error code from the main thread when that list is not empty at the end of
preprocessing.
– Fix the condition used for waiting until all compilation threads finish;
checking that the queue is empty is not enough, since a compilation may be
ongoing. So also track objects whose compilation is ongoing.
---
src/ModelTree.cc | 28 ++++++++++++++++++++--------
src/ModelTree.hh | 6 +++++-
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/src/ModelTree.cc b/src/ModelTree.cc
index 51060f45..1986485b 100644
--- a/src/ModelTree.cc
+++ b/src/ModelTree.cc
@@ -44,7 +44,7 @@
condition_variable_any ModelTree::mex_compilation_cv;
mutex ModelTree::mex_compilation_mut;
vector<tuple<filesystem::path, set<filesystem::path>, string>> ModelTree::mex_compilation_queue;
-set<filesystem::path> ModelTree::mex_compilation_done;
+set<filesystem::path> ModelTree::mex_compilation_ongoing, ModelTree::mex_compilation_done, ModelTree::mex_compilation_failed;
vector<jthread> ModelTree::mex_compilation_workers;
void
@@ -1955,6 +1955,7 @@ ModelTree::initializeMEXCompilationWorkers(int numworkers)
output = get<0>(*it);
cmd = get<2>(*it);
mex_compilation_queue.erase(it);
+ mex_compilation_ongoing.insert(output);
return true;
}
return false;
@@ -1964,13 +1965,13 @@ ModelTree::initializeMEXCompilationWorkers(int numworkers)
if (mex_compilation_cv.wait(lk, stoken, pick_job))
{
lk.unlock();
- if (system(cmd.c_str()))
- {
- cerr << "Compilation failed" << endl;
- exit(EXIT_FAILURE);
- }
+ int r { system(cmd.c_str()) };
lk.lock();
- mex_compilation_done.insert(output);
+ mex_compilation_ongoing.erase(output);
+ if (r)
+ mex_compilation_failed.insert(output);
+ else
+ mex_compilation_done.insert(output);
/* The object just compiled may be a prerequisite for several
other objects, so notify all waiting workers. Also needed to
notify the main thread when in
@@ -1984,7 +1985,18 @@ void
ModelTree::waitForMEXCompilationWorkers()
{
unique_lock<mutex> lk {mex_compilation_mut};
- mex_compilation_cv.wait(lk, [] { return mex_compilation_queue.empty(); });
+ mex_compilation_cv.wait(lk, [] {
+ return (mex_compilation_queue.empty() && mex_compilation_ongoing.empty())
+ || !mex_compilation_failed.empty(); });
+ if (!mex_compilation_failed.empty())
+ {
+ cerr << "Compilation failed for: ";
+ for (const auto &p : mex_compilation_failed)
+ cerr << p.string() << " ";
+ cerr << endl;
+ lk.unlock(); // So that threads can process their stoken
+ exit(EXIT_FAILURE);
+ }
}
void
diff --git a/src/ModelTree.hh b/src/ModelTree.hh
index 84a5d60b..a68bb22b 100644
--- a/src/ModelTree.hh
+++ b/src/ModelTree.hh
@@ -439,8 +439,12 @@ private:
/* Object/MEX files waiting to be compiled (with their prerequisites as 2nd
element and compilation command as the 3rd element) */
static vector<tuple<filesystem::path, set<filesystem::path>, string>> mex_compilation_queue;
- // Object/MEX files already compiled
+ // Object/MEX files in the process of being compiled
+ static set<filesystem::path> mex_compilation_ongoing;
+ // Object/MEX files already compiled successfully
static set<filesystem::path> mex_compilation_done;
+ // Object/MEX files whose compilation failed
+ static set<filesystem::path> mex_compilation_failed;
/* Compute a pseudo-Jacobian whose all elements are either zero or one,
depending on whether the variable symbolically appears in the equation. If
--
GitLab