Parallel build

1 files changed, 320 insertions, 0 deletions

diff --git a/Builder.cpp b/Builder.cpp
new file mode 100644
index 0000000..1f6fbb2
--- /dev/null
+++ b/Builder.cpp
@@ -0,0 +1,320 @@
+#include "Builder.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <iostream>
+#include <iterator>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+#include <fmt/format.h>
+
+#include <libreichwein/file.h>
+#include <libreichwein/stringhelper.h>
+
+namespace fs = std::filesystem;
+namespace pt = boost::property_tree;
+using namespace std::chrono_literals;
+using namespace std::string_literals;
+
+namespace {
+ fs::path get_target(const pt::ptree& ptree) {
+  return ptree.get<std::string>("ymake.build.name");
+ }
+
+ std::vector<fs::path> get_sources(const pt::ptree& ptree) {
+  std::vector<fs::path> sources;
+  for (const pt::ptree::value_type &v: ptree.get_child("ymake.build")) {
+   if (v.first == "source")
+    sources.push_back(v.second.data());
+  }
+  return sources;
+ }
+
+ std::vector<fs::path> get_objects(const pt::ptree& ptree) {
+  std::vector<fs::path> objects{get_sources(ptree)};
+  for (auto &i: objects) {
+   i.replace_extension("o");
+  }
+  return objects;
+ }
+
+ // both need to exist
+ bool is_older(const fs::path& p, const fs::path& other) {
+  auto t_p{fs::last_write_time(p)};
+  auto t_other{fs::last_write_time(other)};
+  return t_p < t_other;
+ }
+
+ // outdated according to dependency file list, non-recursively
+ bool is_outdated(const fs::path& p, const std::vector<fs::path> &dependencies) {
+  if (!fs::exists(p))
+   return true;
+
+  for (const auto& dep: dependencies) {
+   if (!fs::exists(dep) || is_older(p, dep)) {
+    return true;
+   }
+  }
+
+  return false;
+ }
+
+ std::vector<fs::path> deps_from_depfile(const fs::path& path) {
+  std::string depfile_content{Reichwein::File::getFile(path)};
+  std::vector<std::string> parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")};
+  if (parts.size() >= 2) {
+   std::vector<std::string> deps {Reichwein::Stringhelper::split(parts[1], " ")};
+   std::vector<fs::path> result;
+   std::transform(deps.cbegin(), deps.cend(), std::back_inserter(result), [](const std::string& s){ return s; });
+   return result;
+  } else {
+   throw std::runtime_error("Bad depfile contents: "s + path.string());
+  }
+ }
+
+ fs::path depfile_name_from(const fs::path& p) {
+  fs::path depfile{p};
+  depfile.replace_extension("d");
+  return depfile;
+ }
+
+ // return contained dependencies
+ // input: cpp
+ std::vector<fs::path> make_depfile_from(const fs::path& p) {
+  fs::path depfile{depfile_name_from(p)};
+
+  // check if depfile exists and if it contains up to date info
+  if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) {
+   // actually create depfile
+   int result{system(fmt::format("g++ -MM -MF {} -c {}", depfile.string(), p.string()).c_str())};
+   if (result != 0) {
+    throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string()));
+   }
+  }
+
+  return deps_from_depfile(depfile);
+ }
+
+ // type of file can be built from dependencies
+ bool is_buildable_by_extension(const fs::path& p) {
+  fs::path ext{p.extension()};
+  return ext.empty() || ext == ".o";
+ }
+
+ std::unordered_map<fs::path, std::vector<fs::path>> get_dependencies(const pt::ptree& ptree) {
+  std::unordered_map<fs::path, std::vector<fs::path>> dependencies;
+  dependencies.emplace(get_target(ptree), get_objects(ptree));
+  std::vector<fs::path> sources{get_sources(ptree)};
+  for (const auto& p: sources) {
+   fs::path p_obj{p};
+   p_obj.replace_extension("o");
+   std::vector<fs::path> deps {make_depfile_from(p)};
+   // keep .d files for now to speed dependencies detection on following runs
+   //fs::remove(depfile_name_from(p));
+   dependencies.emplace(p_obj, deps);
+  }
+  return dependencies;
+ }
+
+}
+
+Builder::Builder(const pt::ptree& ptree):
+ _target{get_target(ptree)},
+ _objects{get_objects(ptree)},
+ _sources{get_sources(ptree)},
+ _dependencies{get_dependencies(ptree)}
+{
+}
+
+std::vector<fs::path> Builder::dependencies_of(const fs::path& p) {
+ try {
+  return _dependencies.at(p);
+ } catch (const std::out_of_range& ex) {
+  return {}; // empty by default
+ }
+}
+
+// outdated according to dependency tree, recursively
+bool Builder::is_outdated(const fs::path& p) {
+ if (!fs::exists(p))
+  return true;
+
+ std::vector<fs::path> deps{dependencies_of(p)};
+ for (const auto& dep: deps) {
+  if (!fs::exists(dep) || is_older(p, dep)) {
+   return true;
+  }
+
+  if (is_outdated(dep)) {
+   return true;
+  }
+ }
+
+ return false;
+}
+
+// build 1 file
+void Builder::build_file(const fs::path& p) {
+ std::string command;
+
+ if (p.extension() == ".o") {
+   // compile
+   fs::path cppfile{p};
+   cppfile.replace_extension("cpp");
+   command = fmt::format("g++ -std=c++17 -c {} -o {}", cppfile.string(), p.string());
+ } else {
+  // link
+  command = "g++";
+  std::vector<fs::path> objects{dependencies_of(p)};
+  for (auto &i: objects) {
+   command += fmt::format(" {}", i.string());
+  }
+  command += " -lfmt -lreichwein";
+  command += fmt::format(" -o {}", p.string());
+ }
+
+ std::cout << command << std::endl;
+ _runner.spawn(p.string(), command.c_str());
+}
+
+void Builder::cleanup() {
+ std::string path;
+ int exit_code{_runner.wait_one(path)};
+ _activelist.erase(fs::path{path});
+
+ _donelist.insert(fs::path{path});
+ if (exit_code != 0) {
+  throw std::runtime_error(fmt::format("Exit code {}", exit_code));
+ }
+}
+
+// build list of files
+// eats up _buildlist
+void Builder::build_list() {
+
+ if (_buildlist.empty()) {
+  std::cout << "Everything up to date." << std::endl;
+ } else {
+  std::cout << "Running commands: " << std::endl;
+ }
+
+ while (!_buildlist.empty()) {
+  // find file which can be built directly since its build dependencies are up to date
+
+  fs::path current;
+
+  while (current.empty()) {
+   for (auto &i: _buildlist) {
+    std::vector<fs::path> deps{dependencies_of(i)};
+    bool deps_up_to_date{true};
+    for (auto& dep: deps) {
+     if (_activelist.find(dep) != _activelist.end() || is_outdated(dep)) {
+      deps_up_to_date = false;
+     }
+    }
+    if (deps_up_to_date) {
+     current = i;
+     break;
+    }
+   }
+
+   if (current.empty()) {
+    std::this_thread::sleep_for(10ms); // short wait before retry
+    if (_runner.finished() > 0) {
+     cleanup();
+    }
+   }
+  }
+
+  _buildlist.erase(current);
+  _activelist.insert(current);
+
+  while (_runner.full() || _runner.finished() > 0) {
+   cleanup();
+  }
+
+  build_file(current); // calls spawn() on _runner
+ }
+
+ // final cleanup
+ while (_runner.finished() != 0 || _runner.running() != 0) {
+  cleanup();
+ }
+
+ if (!_activelist.empty()) {
+  throw std::runtime_error("Files left actively building");
+ }
+}
+
+// build everything according to specified configuration
+void Builder::build() {
+ std::cout << "Target: " << _target << std::endl;
+
+ std::cout << "Sources: " << std::endl;
+ for (auto &i: _sources) {
+  std::cout << "  " << i << std::endl;
+ }
+
+ // create build list by depth-first search
+ std::cout << "Calculating build list..." << std::endl;
+ std::stack<fs::path> container; // temporary container for search algorithm
+ container.push(_target);
+ while (!container.empty()) {
+  fs::path current{container.top()};
+  container.pop();
+
+  std::vector<fs::path> deps{dependencies_of(current)};
+  for (auto &i: deps) {
+   container.push(i);
+  }
+
+  if (is_outdated(current) && is_buildable_by_extension(current)) {
+   _buildlist.insert(current);
+  }
+ }
+
+ std::cout << "Build list:" << std::endl;
+ for (auto &i: _buildlist) {
+  std::cout << "  " << i << std::endl;
+ }
+
+ build_list();
+}
+
+void Builder::clean() {
+ std::vector<fs::path> cleanlist{_objects};
+ cleanlist.push_back(_target);
+
+ std::vector<std::string> commands;
+ for (auto &i: cleanlist) {
+  if (fs::exists(i)) {
+   commands.push_back(fmt::format("rm -f {}", i.string()));
+  }
+  fs::path depfile{depfile_name_from(i)};
+  if (fs::exists(depfile)) {
+   commands.push_back(fmt::format("rm -f {}", depfile.string()));
+  }
+ }
+
+ std::cout << "Running commands: " << std::endl;
+ for (auto &i: commands) {
+  std::cout << i << std::endl;
+  int result{system(i.c_str())};
+  if (result != 0) {
+   throw std::runtime_error(fmt::format("Error {}", result));
+  }
+ }
+}
+