use_dll: compile MEX files in parallel

Currently two threads are used (one for the dynamic MEX, one for the static
MEX). When the sparse representation is implemented, four threads will be used.

Closes: #41

src/DynareMain.cc

@@ -524,6 +524,11 @@ main(int argc, char **argv)
                            nointeractive, config_file, check_model_changes, minimal_workspace, compute_xrefs,
                            mexext, matlabroot, dynareroot, onlymodel, gui, notime);
 
+  /* Not technically needed since those are std::jthread, but ensures that the
+     preprocessor final message is printed after the end of compilation (and is
+     not printed in case of compilation failure). */
+  ModelTree::joinMEXCompilationThreads();
+
   cout << "Preprocessing completed." << endl;
   return EXIT_SUCCESS;
 }

src/Makefile.am

@@ -70,7 +70,8 @@ EXTRA_DIST = \
 # The -I. is for <FlexLexer.h>
 dynare_preprocessor_CPPFLAGS = $(BOOST_CPPFLAGS) -I.
 dynare_preprocessor_LDFLAGS = $(AM_LDFLAGS) $(BOOST_LDFLAGS)
-dynare_preprocessor_LDADD = macro/libmacro.a
+# -lpthread is no longer necessary for glibc ⩾ 2.34 (i.e. Debian “bookworm” 12)
+dynare_preprocessor_LDADD = macro/libmacro.a -lpthread
 
 # -Ca flag comes from hitting a hard-coded size limit.
 # Partial explanation: https://www.owlfolio.org/possibly-useful/flex-input-scanner-rules-are-too-complicated

src/ModelTree.cc

@@ -36,6 +36,8 @@
 #include <regex>
 #include <utility>
 
+vector<jthread> ModelTree::mex_compilation_threads {};
+
 void
 ModelTree::copyHelper(const ModelTree &m)
 {
@@ -1751,11 +1753,18 @@ ModelTree::compileMEX(const string &basename, const string &funcname, const stri
 
   cout << "Compiling " << funcname << " MEX..." << endl << cmd.str() << endl;
 
-  if (system(cmd.str().c_str()))
+  /* The command line must be captured by value by the thread (a reference
+     would quickly become dangling). And std::ostringstream is not copyable, so
+     capture a std::string. */
+  string cmd_str { cmd.str() };
+  mex_compilation_threads.emplace_back([cmd_str]
+  {
+    if (system(cmd_str.c_str()))
       {
         cerr << "Compilation failed" << endl;
         exit(EXIT_FAILURE);
       }
+  });
 }
 
 void
@@ -1858,3 +1867,10 @@ ModelTree::writeBlockBytecodeAdditionalDerivatives([[maybe_unused]] BytecodeWrit
                                                    [[maybe_unused]] const deriv_node_temp_terms_t &tef_terms) const
 {
 }
+
+void
+ModelTree::joinMEXCompilationThreads()
+{
+  for (auto &it : mex_compilation_threads)
+    it.join();
+}

src/ModelTree.hh

@@ -29,6 +29,7 @@
 #include <filesystem>
 #include <optional>
 #include <cassert>
+#include <thread>
 
 #include "DataTree.hh"
 #include "EquationTags.hh"
@@ -324,6 +325,9 @@ protected:
   /*! Maps endogenous type specific IDs to equation numbers */
   vector<int> endo2eq;
 
+  // Stores threads for compiling MEX files in parallel
+  static vector<jthread> mex_compilation_threads;
+
   /* Compute a pseudo-Jacobian whose all elements are either zero or one,
      depending on whether the variable symbolically appears in the equation */
   jacob_map_t computeSymbolicJacobian() const;
@@ -459,7 +463,12 @@ private:
   //! Finds a suitable GCC compiler on macOS
   static string findGccOnMacos(const string &mexext);
 #endif
-  //! Compiles a MEX file
+  /* Compiles a MEX file. The compilation is done in a separate asynchronous
+     thread, so the call to this function is not blocking.
+     TODO: further improve the function so that when a MEX has multiple source
+     files, those get compiled in separate threads; this could however
+     require implementing a scheduler, so as to not run more threads than
+     there are logical cores. */
   void compileMEX(const string &basename, const string &funcname, const string &mexext, const vector<filesystem::path> &src_files, const filesystem::path &matlabroot, const filesystem::path &dynareroot) const;
 
 public:
@@ -507,6 +516,9 @@ public:
      If no such equation can be found, throws an ExprNode::MatchFailureExpression */
   expr_t getRHSFromLHS(expr_t lhs) const;
 
+  // Calls join() on all MEX compilation threads
+  static void joinMEXCompilationThreads();
+
   //! Returns all the equation tags associated to an equation
   map<string, string>
   getEquationTags(int eq) const