From 6a0de236e8e3f3b46aadcb8b1c67a58c2b289312 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 30 Sep 2022 15:39:41 +0200
Subject: [PATCH] use_dll: compile MEX files in parallel
Currently two threads are used (one for the dynamic MEX, one for the static
MEX). When the sparse representation is implemented, four threads will be used.
Closes: #41
---
src/DynareMain.cc | 5 +++++
src/ModelTree.cc | 26 +++++++++++++++++++++-----
src/ModelTree.hh | 14 +++++++++++++-
3 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/src/DynareMain.cc b/src/DynareMain.cc
index f2d6182b..aa7677ea 100644
--- a/src/DynareMain.cc
+++ b/src/DynareMain.cc
@@ -524,6 +524,11 @@ main(int argc, char **argv)
nointeractive, config_file, check_model_changes, minimal_workspace, compute_xrefs,
mexext, matlabroot, dynareroot, onlymodel, gui, notime);
+ /* Not technically needed since those are std::jthread, but ensures that the
+ preprocessor final message is printed after the end of compilation (and is
+ not printed in case of compilation failure). */
+ ModelTree::joinMEXCompilationThreads();
+
cout << "Preprocessing completed." << endl;
return EXIT_SUCCESS;
}
diff --git a/src/ModelTree.cc b/src/ModelTree.cc
index affb101a..364a1777 100644
--- a/src/ModelTree.cc
+++ b/src/ModelTree.cc
@@ -36,6 +36,8 @@
#include <regex>
#include <utility>
+vector<jthread> ModelTree::mex_compilation_threads {};
+
void
ModelTree::copyHelper(const ModelTree &m)
{
@@ -1751,11 +1753,18 @@ ModelTree::compileMEX(const string &basename, const string &funcname, const stri
cout << "Compiling " << funcname << " MEX..." << endl << cmd.str() << endl;
- if (system(cmd.str().c_str()))
- {
- cerr << "Compilation failed" << endl;
- exit(EXIT_FAILURE);
- }
+ /* The command line must be captured by value by the thread (a reference
+ would quickly become dangling). And std::ostringstream is not copyable, so
+ capture a std::string. */
+ string cmd_str { cmd.str() };
+ mex_compilation_threads.emplace_back([cmd_str]
+ {
+ if (system(cmd_str.c_str()))
+ {
+ cerr << "Compilation failed" << endl;
+ exit(EXIT_FAILURE);
+ }
+ });
}
void
@@ -1858,3 +1867,10 @@ ModelTree::writeBlockBytecodeAdditionalDerivatives([[maybe_unused]] BytecodeWrit
[[maybe_unused]] const deriv_node_temp_terms_t &tef_terms) const
{
}
+
+void
+ModelTree::joinMEXCompilationThreads()
+{
+ for (auto &it : mex_compilation_threads)
+ it.join();
+}
diff --git a/src/ModelTree.hh b/src/ModelTree.hh
index 9cea1fbf..eba02be2 100644
--- a/src/ModelTree.hh
+++ b/src/ModelTree.hh
@@ -29,6 +29,7 @@
#include <filesystem>
#include <optional>
#include <cassert>
+#include <thread>
#include "DataTree.hh"
#include "EquationTags.hh"
@@ -324,6 +325,9 @@ protected:
/*! Maps endogenous type specific IDs to equation numbers */
vector<int> endo2eq;
+ // Stores threads for compiling MEX files in parallel
+ static vector<jthread> mex_compilation_threads;
+
/* Compute a pseudo-Jacobian whose all elements are either zero or one,
depending on whether the variable symbolically appears in the equation */
jacob_map_t computeSymbolicJacobian() const;
@@ -459,7 +463,12 @@ private:
//! Finds a suitable GCC compiler on macOS
static string findGccOnMacos(const string &mexext);
#endif
- //! Compiles a MEX file
+ /* Compiles a MEX file. The compilation is done in a separate asynchronous
+ thread, so the call to this function is not blocking.
+ TODO: further improve the function so that when a MEX has multiple source
+ files, those get compiled in separate threads; this could however
+ require implementing a scheduler, so as to not run more threads than
+ there are logical cores. */
void compileMEX(const string &basename, const string &funcname, const string &mexext, const vector<filesystem::path> &src_files, const filesystem::path &matlabroot, const filesystem::path &dynareroot) const;
public:
@@ -507,6 +516,9 @@ public:
If no such equation can be found, throws an ExprNode::MatchFailureExpression */
expr_t getRHSFromLHS(expr_t lhs) const;
+ // Calls join() on all MEX compilation threads
+ static void joinMEXCompilationThreads();
+
//! Returns all the equation tags associated to an equation
map<string, string>
getEquationTags(int eq) const
--
GitLab