From 45983abe664be648b513202c8c12578c9a85784f Mon Sep 17 00:00:00 2001 From: Roland Reichwein Date: Fri, 3 May 2024 21:03:06 +0200 Subject: Parallel build --- Builder.cpp | 320 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 Builder.cpp (limited to 'Builder.cpp') diff --git a/Builder.cpp b/Builder.cpp new file mode 100644 index 0000000..1f6fbb2 --- /dev/null +++ b/Builder.cpp @@ -0,0 +1,320 @@ +#include "Builder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +namespace fs = std::filesystem; +namespace pt = boost::property_tree; +using namespace std::chrono_literals; +using namespace std::string_literals; + +namespace { + fs::path get_target(const pt::ptree& ptree) { + return ptree.get("ymake.build.name"); + } + + std::vector get_sources(const pt::ptree& ptree) { + std::vector sources; + for (const pt::ptree::value_type &v: ptree.get_child("ymake.build")) { + if (v.first == "source") + sources.push_back(v.second.data()); + } + return sources; + } + + std::vector get_objects(const pt::ptree& ptree) { + std::vector objects{get_sources(ptree)}; + for (auto &i: objects) { + i.replace_extension("o"); + } + return objects; + } + + // both need to exist + bool is_older(const fs::path& p, const fs::path& other) { + auto t_p{fs::last_write_time(p)}; + auto t_other{fs::last_write_time(other)}; + return t_p < t_other; + } + + // outdated according to dependency file list, non-recursively + bool is_outdated(const fs::path& p, const std::vector &dependencies) { + if (!fs::exists(p)) + return true; + + for (const auto& dep: dependencies) { + if (!fs::exists(dep) || is_older(p, dep)) { + return true; + } + } + + return false; + } + + std::vector deps_from_depfile(const fs::path& path) { + std::string depfile_content{Reichwein::File::getFile(path)}; + std::vector parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")}; + if (parts.size() >= 2) { + std::vector deps {Reichwein::Stringhelper::split(parts[1], " ")}; + std::vector result; + std::transform(deps.cbegin(), deps.cend(), std::back_inserter(result), [](const std::string& s){ return s; }); + return result; + } else { + throw std::runtime_error("Bad depfile contents: "s + path.string()); + } + } + + fs::path depfile_name_from(const fs::path& p) { + fs::path depfile{p}; + depfile.replace_extension("d"); + return depfile; + } + + // return contained dependencies + // input: cpp + std::vector make_depfile_from(const fs::path& p) { + fs::path depfile{depfile_name_from(p)}; + + // check if depfile exists and if it contains up to date info + if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) { + // actually create depfile + int result{system(fmt::format("g++ -MM -MF {} -c {}", depfile.string(), p.string()).c_str())}; + if (result != 0) { + throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string())); + } + } + + return deps_from_depfile(depfile); + } + + // type of file can be built from dependencies + bool is_buildable_by_extension(const fs::path& p) { + fs::path ext{p.extension()}; + return ext.empty() || ext == ".o"; + } + + std::unordered_map> get_dependencies(const pt::ptree& ptree) { + std::unordered_map> dependencies; + dependencies.emplace(get_target(ptree), get_objects(ptree)); + std::vector sources{get_sources(ptree)}; + for (const auto& p: sources) { + fs::path p_obj{p}; + p_obj.replace_extension("o"); + std::vector deps {make_depfile_from(p)}; + // keep .d files for now to speed dependencies detection on following runs + //fs::remove(depfile_name_from(p)); + dependencies.emplace(p_obj, deps); + } + return dependencies; + } + +} + +Builder::Builder(const pt::ptree& ptree): + _target{get_target(ptree)}, + _objects{get_objects(ptree)}, + _sources{get_sources(ptree)}, + _dependencies{get_dependencies(ptree)} +{ +} + +std::vector Builder::dependencies_of(const fs::path& p) { + try { + return _dependencies.at(p); + } catch (const std::out_of_range& ex) { + return {}; // empty by default + } +} + +// outdated according to dependency tree, recursively +bool Builder::is_outdated(const fs::path& p) { + if (!fs::exists(p)) + return true; + + std::vector deps{dependencies_of(p)}; + for (const auto& dep: deps) { + if (!fs::exists(dep) || is_older(p, dep)) { + return true; + } + + if (is_outdated(dep)) { + return true; + } + } + + return false; +} + +// build 1 file +void Builder::build_file(const fs::path& p) { + std::string command; + + if (p.extension() == ".o") { + // compile + fs::path cppfile{p}; + cppfile.replace_extension("cpp"); + command = fmt::format("g++ -std=c++17 -c {} -o {}", cppfile.string(), p.string()); + } else { + // link + command = "g++"; + std::vector objects{dependencies_of(p)}; + for (auto &i: objects) { + command += fmt::format(" {}", i.string()); + } + command += " -lfmt -lreichwein"; + command += fmt::format(" -o {}", p.string()); + } + + std::cout << command << std::endl; + _runner.spawn(p.string(), command.c_str()); +} + +void Builder::cleanup() { + std::string path; + int exit_code{_runner.wait_one(path)}; + _activelist.erase(fs::path{path}); + + _donelist.insert(fs::path{path}); + if (exit_code != 0) { + throw std::runtime_error(fmt::format("Exit code {}", exit_code)); + } +} + +// build list of files +// eats up _buildlist +void Builder::build_list() { + + if (_buildlist.empty()) { + std::cout << "Everything up to date." << std::endl; + } else { + std::cout << "Running commands: " << std::endl; + } + + while (!_buildlist.empty()) { + // find file which can be built directly since its build dependencies are up to date + + fs::path current; + + while (current.empty()) { + for (auto &i: _buildlist) { + std::vector deps{dependencies_of(i)}; + bool deps_up_to_date{true}; + for (auto& dep: deps) { + if (_activelist.find(dep) != _activelist.end() || is_outdated(dep)) { + deps_up_to_date = false; + } + } + if (deps_up_to_date) { + current = i; + break; + } + } + + if (current.empty()) { + std::this_thread::sleep_for(10ms); // short wait before retry + if (_runner.finished() > 0) { + cleanup(); + } + } + } + + _buildlist.erase(current); + _activelist.insert(current); + + while (_runner.full() || _runner.finished() > 0) { + cleanup(); + } + + build_file(current); // calls spawn() on _runner + } + + // final cleanup + while (_runner.finished() != 0 || _runner.running() != 0) { + cleanup(); + } + + if (!_activelist.empty()) { + throw std::runtime_error("Files left actively building"); + } +} + +// build everything according to specified configuration +void Builder::build() { + std::cout << "Target: " << _target << std::endl; + + std::cout << "Sources: " << std::endl; + for (auto &i: _sources) { + std::cout << " " << i << std::endl; + } + + // create build list by depth-first search + std::cout << "Calculating build list..." << std::endl; + std::stack container; // temporary container for search algorithm + container.push(_target); + while (!container.empty()) { + fs::path current{container.top()}; + container.pop(); + + std::vector deps{dependencies_of(current)}; + for (auto &i: deps) { + container.push(i); + } + + if (is_outdated(current) && is_buildable_by_extension(current)) { + _buildlist.insert(current); + } + } + + std::cout << "Build list:" << std::endl; + for (auto &i: _buildlist) { + std::cout << " " << i << std::endl; + } + + build_list(); +} + +void Builder::clean() { + std::vector cleanlist{_objects}; + cleanlist.push_back(_target); + + std::vector commands; + for (auto &i: cleanlist) { + if (fs::exists(i)) { + commands.push_back(fmt::format("rm -f {}", i.string())); + } + fs::path depfile{depfile_name_from(i)}; + if (fs::exists(depfile)) { + commands.push_back(fmt::format("rm -f {}", depfile.string())); + } + } + + std::cout << "Running commands: " << std::endl; + for (auto &i: commands) { + std::cout << i << std::endl; + int result{system(i.c_str())}; + if (result != 0) { + throw std::runtime_error(fmt::format("Error {}", result)); + } + } +} + -- cgit v1.2.3