From ec80e861353b0da535a2913bd8adbdda4cbab4e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 7 Jul 2023 14:44:16 +0200
Subject: [PATCH] Disable use_dll option
std::jthread and std::stop_token are not supported by LLVM/libc++ < 17.
---
src/DynareMain.cc | 10 --
src/ModelTree.cc | 264 ++--------------------------------------------
src/ModelTree.hh | 26 -----
3 files changed, 8 insertions(+), 292 deletions(-)
diff --git a/src/DynareMain.cc b/src/DynareMain.cc
index 4e9ff55a..76ec0091 100644
--- a/src/DynareMain.cc
+++ b/src/DynareMain.cc
@@ -508,10 +508,6 @@ main(int argc, char** argv)
exit(EXIT_FAILURE);
}
- if (mod_file->use_dll)
- ModelTree::initializeMEXCompilationWorkers(max(jthread::hardware_concurrency(), 1U), dynareroot,
- mexext);
-
if (json == JsonOutputPointType::parsing)
mod_file->writeJsonOutput(basename, json, json_output_mode, onlyjson);
@@ -543,12 +539,6 @@ main(int argc, char** argv)
nointeractive, config, check_model_changes, minimal_workspace,
compute_xrefs, mexext, matlabroot, onlymodel, gui, notime);
- /* Ensures that workers are not destroyed before they finish compiling.
- Also ensures that the preprocessor final message is printed after the end of
- compilation (and is not printed in case of compilation failure). */
- if (mod_file->use_dll)
- ModelTree::waitForMEXCompilationWorkers();
-
cout << "Preprocessing completed." << endl;
return EXIT_SUCCESS;
}
diff --git a/src/ModelTree.cc b/src/ModelTree.cc
index 533a13ce..1138a740 100644
--- a/src/ModelTree.cc
+++ b/src/ModelTree.cc
@@ -38,17 +38,6 @@
#include <regex>
#include <utility>
-/* NB: The workers must be listed *after* all the other static variables
- related to MEX compilation, so that when the preprocessor exits, the workers
- are destroyed *before* those variables (since the former rely on the latter
- for their functioning). */
-condition_variable_any ModelTree::mex_compilation_cv;
-mutex ModelTree::mex_compilation_mut;
-vector<tuple<filesystem::path, set<filesystem::path>, string>> ModelTree::mex_compilation_queue;
-set<filesystem::path> ModelTree::mex_compilation_ongoing, ModelTree::mex_compilation_done,
- ModelTree::mex_compilation_failed;
-vector<jthread> ModelTree::mex_compilation_workers;
-
void
ModelTree::copyHelper(const ModelTree& m)
{
@@ -1700,135 +1689,15 @@ ModelTree::findCompilerOnMacos(const string& mexext)
#endif
filesystem::path
-ModelTree::compileMEX(const filesystem::path& output_dir, const string& output_basename,
- const string& mexext, const vector<filesystem::path>& input_files,
- const filesystem::path& matlabroot, bool link) const
+ModelTree::compileMEX([[maybe_unused]] const filesystem::path& output_dir,
+ [[maybe_unused]] const string& output_basename,
+ [[maybe_unused]] const string& mexext,
+ [[maybe_unused]] const vector<filesystem::path>& input_files,
+ [[maybe_unused]] const filesystem::path& matlabroot,
+ [[maybe_unused]] bool link) const
{
- assert(!mex_compilation_workers.empty());
-
- const string gcc_opt_flags {
- "-O3 -g0 --param ira-max-conflict-table-size=1 -fno-forward-propagate -fno-gcse -fno-dce "
- "-fno-dse -fno-tree-fre -fno-tree-pre -fno-tree-cselim -fno-tree-dse -fno-tree-dce "
- "-fno-tree-pta -fno-gcse-after-reload"};
- const string clang_opt_flags {
- "-O3 -g0 --param ira-max-conflict-table-size=1 -Wno-unused-command-line-argument"};
-
- filesystem::path compiler;
- ostringstream flags;
- string libs;
- bool is_clang {false};
-
- if (matlabroot.empty())
- {
- cerr << "ERROR: 'matlabroot' option to preprocessor is not set, needed with 'use_dll'"
- << endl;
- exit(EXIT_FAILURE);
- }
-
- if (mexext == "mex")
- {
- // Octave
- compiler = matlabroot / "bin" / "mkoctfile";
- flags << "--mex";
- }
- else
- {
- // MATLAB
- compiler = "gcc";
- string arch = matlab_arch(mexext);
- auto include_dir = matlabroot / "extern" / "include";
- flags << "-I " << include_dir;
- auto bin_dir = matlabroot / "bin" / arch;
- flags << " -L " << bin_dir;
- flags << " -fexceptions -DNDEBUG";
- libs = "-lmex -lmx";
- if (mexext == "mexa64")
- {
- // GNU/Linux
- flags << " -D_GNU_SOURCE -fPIC -pthread"
- << " -shared -Wl,--no-undefined -Wl,-rpath-link," << bin_dir;
- libs += " -lm";
- }
- else if (mexext == "mexw64") // Windows
- flags << " -static-libgcc -shared";
-#ifdef __APPLE__
- else if (mexext == "mexmaci64" || mexext == "mexmaca64")
- {
- tie(compiler, is_clang) = findCompilerOnMacos(mexext);
- flags << " -fno-common -Wl,-twolevel_namespace -undefined error -bundle";
- libs += " -lm";
- }
-#endif
- else
- {
- cerr << "ERROR: unsupported value '" << mexext << "' for 'mexext' option" << endl;
- exit(EXIT_FAILURE);
- }
- }
-
- filesystem::path output_filename {output_dir / (output_basename + "." + (link ? mexext : "o"))};
-
- ostringstream cmd;
-
-#ifdef _WIN32
- /* On Windows, system() hands the command over to "cmd.exe /C". We need to
- enclose the whole command line within double quotes if we want the inner
- quotes to be correctly handled. See "cmd /?" for more details. */
- cmd << '"';
-#endif
-
- if (user_set_compiler.empty())
- cmd << compiler << " ";
- else if (!filesystem::exists(user_set_compiler))
- {
- cerr << "Error: The specified compiler '" << user_set_compiler
- << "' cannot be found on your system" << endl;
- exit(EXIT_FAILURE);
- }
- else
- cmd << user_set_compiler << " ";
-
- if (user_set_subst_flags.empty())
- cmd << (is_clang ? clang_opt_flags : gcc_opt_flags) << " " << flags.str() << " ";
- else
- cmd << user_set_subst_flags << " ";
-
- if (!user_set_add_flags.empty())
- cmd << user_set_add_flags << " ";
-
- for (auto& f : input_files)
- cmd << f << " ";
- cmd << "-o " << output_filename << " ";
-
- if (link)
- {
- if (user_set_subst_libs.empty())
- cmd << libs;
- else
- cmd << user_set_subst_libs;
- if (!user_set_add_libs.empty())
- cmd << " " << user_set_add_libs;
- }
- else
- cmd << " -c";
-
-#ifdef _WIN32
- cmd << '"';
-#endif
-
- cout << "Compiling " << output_filename.string() << endl;
-
- // The prerequisites are the object files among the input files
- set<filesystem::path> prerequisites;
- copy_if(input_files.begin(), input_files.end(), inserter(prerequisites, prerequisites.end()),
- [](const auto& p) { return p.extension() == ".o"; });
-
- unique_lock<mutex> lk {mex_compilation_mut};
- mex_compilation_queue.emplace_back(output_filename, prerequisites, cmd.str());
- lk.unlock();
- mex_compilation_cv.notify_one();
-
- return output_filename;
+ // Do nothing
+ return {};
}
void
@@ -1954,123 +1823,6 @@ ModelTree::getRHSFromLHS(expr_t lhs) const
throw ExprNode::MatchFailureException {"Cannot find an equation with the requested LHS"};
}
-void
-ModelTree::initializeMEXCompilationWorkers(int numworkers, const filesystem::path& dynareroot,
- const string& mexext)
-{
- assert(numworkers > 0);
- assert(mex_compilation_workers.empty());
-
- cout << "Spawning " << numworkers << " threads for compiling MEX files." << endl;
-
- for (int i {0}; i < numworkers; i++)
- /* Passing the stop_token by const reference is ok (and makes clang-tidy happier),
- since the std::jthread constructor calls the lambda with the return argument of the
- get_stop_token() method, which returns a stop_token by value; hence there is no lifetime
- issue. See:
- https://stackoverflow.com/questions/72990607/const-stdstop-token-or-just-stdstop-token-as-parameter-for-thread-funct
- */
- mex_compilation_workers.emplace_back([](const stop_token& stoken) {
- unique_lock<mutex> lk {mex_compilation_mut};
- filesystem::path output;
- string cmd;
-
- /* Look for an object to compile, whose prerequisites are already
- compiled. If found, remove it from the queue, save the output path and
- the compilation command, and return true. Must be run under the lock. */
- auto pick_job = [&cmd, &output] {
- for (auto it {mex_compilation_queue.begin()}; it != mex_compilation_queue.end(); ++it)
- if (const auto& prerequisites {get<1>(*it)}; // Will become dangling after erase
- includes(mex_compilation_done.begin(), mex_compilation_done.end(),
- prerequisites.begin(), prerequisites.end()))
- {
- output = get<0>(*it);
- cmd = get<2>(*it);
- mex_compilation_queue.erase(it);
- mex_compilation_ongoing.insert(output);
- return true;
- }
- return false;
- };
-
- while (!stoken.stop_requested())
- if (mex_compilation_cv.wait(lk, stoken, pick_job))
- {
- lk.unlock();
- int r {system(cmd.c_str())};
- lk.lock();
- mex_compilation_ongoing.erase(output);
- if (r)
- mex_compilation_failed.insert(output);
- else
- mex_compilation_done.insert(output);
- /* The object just compiled may be a prerequisite for several
- other objects, so notify all waiting workers. Also needed to
- notify the main thread when in
- ModelTree::waitForMEXCompilationWorkers().*/
- mex_compilation_cv.notify_all();
- }
- });
-
- /* Set some environment variables needed for compilation on Windows/MATLAB
- and macOS/Octave.
- For Windows/MATLAB, this should be done only once, because otherwise
- the PATH variable can become too long and GCC will not be found. */
- if (mexext == "mexw64")
- {
- // Put the MinGW environment shipped with Dynare in the path
- auto mingwpath = dynareroot / "mingw64" / "bin";
- string newpath = "PATH=" + mingwpath.string() + ';' + getenv("PATH");
- /* We can’t use setenv() since it is not available on MinGW. Note that
- putenv() seems to make an internal copy of the string on MinGW,
- contrary to what is done on GNU/Linux and macOS. */
- if (putenv(const_cast<char*>(newpath.c_str())) != 0)
- {
- cerr << "Can't set PATH" << endl;
- exit(EXIT_FAILURE);
- }
- }
-#ifdef __APPLE__
- else if (mexext == "mex")
- {
- /* On macOS, with Octave, enforce our compiler. In particular this is
- necessary if we’ve selected GCC; otherwise Clang will be used, and
- it does not accept the same optimization flags (see dynare#1797) */
- auto [compiler_path, is_clang] {findCompilerOnMacos(mexext)};
- if (setenv("CC", compiler_path.c_str(), 1) != 0)
- {
- cerr << "Can't set CC environment variable" << endl;
- exit(EXIT_FAILURE);
- }
- // We also define CXX, because that is used for linking
- if (setenv("CXX", compiler_path.c_str(), 1) != 0)
- {
- cerr << "Can't set CXX environment variable" << endl;
- exit(EXIT_FAILURE);
- }
- }
-#endif
-}
-
-void
-ModelTree::waitForMEXCompilationWorkers()
-{
- unique_lock<mutex> lk {mex_compilation_mut};
- mex_compilation_cv.wait(lk, [] {
- return (mex_compilation_queue.empty() && mex_compilation_ongoing.empty())
- || !mex_compilation_failed.empty();
- });
- if (!mex_compilation_failed.empty())
- {
- cerr << "Compilation failed for: ";
- for (const auto& p : mex_compilation_failed)
- cerr << p.string() << " ";
- cerr << endl;
- lk.unlock(); // So that threads can process their stoken
- exit(EXIT_FAILURE);
- }
-}
-
void
ModelTree::computingPassBlock(const eval_context_t& eval_context, bool no_tmp_terms)
{
diff --git a/src/ModelTree.hh b/src/ModelTree.hh
index f6247744..d9e44c77 100644
--- a/src/ModelTree.hh
+++ b/src/ModelTree.hh
@@ -450,24 +450,6 @@ private:
/*! Maps endogenous type specific IDs to equation numbers */
vector<int> endo2eq;
- // Stores workers used for compiling MEX files in parallel
- static vector<jthread> mex_compilation_workers;
-
- /* The following variables implement the thread synchronization mechanism for
- limiting the number of concurrent GCC processes and tracking dependencies
- between object files. */
- static condition_variable_any mex_compilation_cv;
- static mutex mex_compilation_mut;
- /* Object/MEX files waiting to be compiled (with their prerequisites as 2nd
- element and compilation command as the 3rd element) */
- static vector<tuple<filesystem::path, set<filesystem::path>, string>> mex_compilation_queue;
- // Object/MEX files in the process of being compiled
- static set<filesystem::path> mex_compilation_ongoing;
- // Object/MEX files already compiled successfully
- static set<filesystem::path> mex_compilation_done;
- // Object/MEX files whose compilation failed
- static set<filesystem::path> mex_compilation_failed;
-
/* Compute a pseudo-Jacobian whose all elements are either zero or one,
depending on whether the variable symbolically appears in the equation. If
contemporaneous_only=true, only considers contemporaneous occurences of
@@ -701,14 +683,6 @@ public:
If no such equation can be found, throws an ExprNode::MatchFailureExpression */
expr_t getRHSFromLHS(expr_t lhs) const;
- /* Initialize the MEX compilation workers (and some environment variables
- needed for finding GCC) */
- static void initializeMEXCompilationWorkers(int numworkers, const filesystem::path& dynareroot,
- const string& mexext);
-
- // Waits until the MEX compilation queue is empty
- static void waitForMEXCompilationWorkers();
-
// Write the definitions of the auxiliary variables (assumed to be in recursive order)
void writeAuxVarRecursiveDefinitions(ostream& output, ExprNodeOutputType output_type) const;
--
GitLab