From 45983abe664be648b513202c8c12578c9a85784f Mon Sep 17 00:00:00 2001 From: Roland Reichwein Date: Fri, 3 May 2024 21:03:06 +0200 Subject: Parallel build --- builder.cpp | 296 ------------------------------------------------------------ 1 file changed, 296 deletions(-) delete mode 100644 builder.cpp (limited to 'builder.cpp') diff --git a/builder.cpp b/builder.cpp deleted file mode 100644 index ee9c16b..0000000 --- a/builder.cpp +++ /dev/null @@ -1,296 +0,0 @@ -#include "builder.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include - -namespace fs = std::filesystem; -namespace bp = boost::process; -namespace pt = boost::property_tree; -using namespace std::string_literals; - -namespace { - fs::path get_target(const pt::ptree& ptree) { - return ptree.get("ymake.build.name"); - } - - std::vector get_sources(const pt::ptree& ptree) { - std::vector sources; - for (const pt::ptree::value_type &v: ptree.get_child("ymake.build")) { - if (v.first == "source") - sources.push_back(v.second.data()); - } - return sources; - } - - std::vector get_objects(const pt::ptree& ptree) { - std::vector objects{get_sources(ptree)}; - for (auto &i: objects) { - i.replace_extension("o"); - } - return objects; - } - - // both need to exist - bool is_older(const fs::path& p, const fs::path& other) { - auto t_p{fs::last_write_time(p)}; - auto t_other{fs::last_write_time(other)}; - return t_p < t_other; - } - - // outdated according to dependency file list, non-recursively - bool is_outdated(const fs::path& p, const std::vector &dependencies) { - if (!fs::exists(p)) - return true; - - for (const auto& dep: dependencies) { - if (!fs::exists(dep) || is_older(p, dep)) { - return true; - } - } - - return false; - } - - std::vector deps_from_depfile(const fs::path& path) { - std::string depfile_content{Reichwein::File::getFile(path)}; - std::vector parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")}; - if (parts.size() >= 2) { - std::vector deps {Reichwein::Stringhelper::split(parts[1], " ")}; - std::vector result; - std::transform(deps.cbegin(), deps.cend(), std::back_inserter(result), [](const std::string& s){ return s; }); - return result; - } else { - throw std::runtime_error("Bad depfile contents: "s + path.string()); - } - } - - fs::path depfile_name_from(const fs::path& p) { - fs::path depfile{p}; - depfile.replace_extension("d"); - return depfile; - } - - // return contained dependencies - // input: cpp - std::vector make_depfile_from(const fs::path& p) { - fs::path depfile{depfile_name_from(p)}; - - // check if depfile exists and if it contains up to date info - if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) { - // actually create depfile - int result{system(fmt::format("g++ -MM -MF {} -c {}", depfile.string(), p.string()).c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string())); - } - } - - return deps_from_depfile(depfile); - } - - // type of file can be built from dependencies - bool is_buildable_by_extension(const fs::path& p) { - fs::path ext{p.extension()}; - return ext.empty() || ext == ".o"; - } - - std::unordered_map> get_dependencies(const pt::ptree& ptree) { - std::unordered_map> dependencies; - dependencies.emplace(get_target(ptree), get_objects(ptree)); - std::vector sources{get_sources(ptree)}; - for (const auto& p: sources) { - fs::path p_obj{p}; - p_obj.replace_extension("o"); - std::vector deps {make_depfile_from(p)}; - // keep .d files for now to speed dependencies detection on following runs - //fs::remove(depfile_name_from(p)); - dependencies.emplace(p_obj, deps); - } - return dependencies; - } - -} - -Builder::Builder(const pt::ptree& ptree): - _target{get_target(ptree)}, - _objects{get_objects(ptree)}, - _sources{get_sources(ptree)}, - _dependencies{get_dependencies(ptree)} -{ -} - -std::vector Builder::dependencies_of(const fs::path& p) { - try { - return _dependencies.at(p); - } catch (const std::out_of_range& ex) { - return {}; // empty by default - } -} - -// outdated according to dependency tree, recursively -bool Builder::is_outdated(const fs::path& p) { - if (!fs::exists(p)) - return true; - - std::vector deps{dependencies_of(p)}; - for (const auto& dep: deps) { - if (!fs::exists(dep) || is_older(p, dep)) { - return true; - } - - if (is_outdated(dep)) { - return true; - } - } - - return false; -} - -// build 1 file -void Builder::build(const fs::path& p) { - std::string command; - - if (p.extension() == ".o") { - // compile - fs::path cppfile{p}; - cppfile.replace_extension("cpp"); - command = fmt::format("g++ -std=c++17 -c {} -o {}", cppfile.string(), p.string()); - } else { - // link - command = "g++"; - std::vector objects{dependencies_of(p)}; - for (auto &i: objects) { - command += fmt::format(" {}", i.string()); - } - command += " -lfmt -lreichwein"; - command += fmt::format(" -o {}", p.string()); - } - - std::cout << command << std::endl; - int result{system(command.c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Error {}", result)); - } -} - -// build list of files -// eats up input -void Builder::build(std::unordered_set& buildlist) { - std::unordered_set activelist; // currently building - std::unordered_set donelist; // build done - - if (buildlist.empty()) { - std::cout << "Everything up to date." << std::endl; - } else { - std::cout << "Running commands: " << std::endl; - } - - while (!buildlist.empty()) { - // find file which can be built directly since its build dependencies are up to date - // this holds for either any member of the set (e.g. at begin()), or any of its direct or indirect dependencies - fs::path current {*buildlist.begin()}; - - bool found_outdated{}; // outdated dependencies - - do { - found_outdated = false; - std::vector deps{dependencies_of(current)}; - for (auto& i: deps) { - if (activelist.find(i) == activelist.end() && is_outdated(i)) { - found_outdated = true; - current = i; - break; - } - } - } while (found_outdated); - - buildlist.erase(current); - - activelist.insert(current); - // TODO: build in background - build(current); - activelist.erase(current); - - donelist.insert(current); - } -} - -// build everything according to specified configuration -void Builder::build() { - std::cout << "Target: " << _target << std::endl; - - std::cout << "Sources: " << std::endl; - for (auto &i: _sources) { - std::cout << " " << i << std::endl; - } - - // create build list by depth-first search - std::cout << "Calculating build list..." << std::endl; - std::unordered_set buildlist; - std::stack container; // temporary container for search algorithm - container.push(_target); - while (!container.empty()) { - fs::path current{container.top()}; - container.pop(); - - std::vector deps{dependencies_of(current)}; - for (auto &i: deps) { - container.push(i); - } - - if (is_outdated(current) && is_buildable_by_extension(current)) { - buildlist.insert(current); - } - } - - std::cout << "Build list:" << std::endl; - for (auto &i: buildlist) { - std::cout << " " << i << std::endl; - } - - build(buildlist); -} - -void Builder::clean() { - std::vector cleanlist{_objects}; - cleanlist.push_back(_target); - - std::vector commands; - for (auto &i: cleanlist) { - if (fs::exists(i)) { - commands.push_back(fmt::format("rm -f {}", i.string())); - } - fs::path depfile{depfile_name_from(i)}; - if (fs::exists(depfile)) { - commands.push_back(fmt::format("rm -f {}", depfile.string())); - } - } - - std::cout << "Running commands: " << std::endl; - for (auto &i: commands) { - std::cout << i << std::endl; - int result{system(i.c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Error {}", result)); - } - } -} - -- cgit v1.2.3