From 45983abe664be648b513202c8c12578c9a85784f Mon Sep 17 00:00:00 2001 From: Roland Reichwein Date: Fri, 3 May 2024 21:03:06 +0200 Subject: Parallel build --- Builder.cpp | 320 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Builder.h | 40 +++++++ Makefile | 2 +- ProcessRunner.cpp | 92 ++++++++++++++++ ProcessRunner.h | 36 ++++++ YMakefile | 3 +- builder.cpp | 296 -------------------------------------------------- builder.h | 29 ----- ymake.cpp | 2 +- 9 files changed, 492 insertions(+), 328 deletions(-) create mode 100644 Builder.cpp create mode 100644 Builder.h create mode 100644 ProcessRunner.cpp create mode 100644 ProcessRunner.h delete mode 100644 builder.cpp delete mode 100644 builder.h diff --git a/Builder.cpp b/Builder.cpp new file mode 100644 index 0000000..1f6fbb2 --- /dev/null +++ b/Builder.cpp @@ -0,0 +1,320 @@ +#include "Builder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +namespace fs = std::filesystem; +namespace pt = boost::property_tree; +using namespace std::chrono_literals; +using namespace std::string_literals; + +namespace { + fs::path get_target(const pt::ptree& ptree) { + return ptree.get("ymake.build.name"); + } + + std::vector get_sources(const pt::ptree& ptree) { + std::vector sources; + for (const pt::ptree::value_type &v: ptree.get_child("ymake.build")) { + if (v.first == "source") + sources.push_back(v.second.data()); + } + return sources; + } + + std::vector get_objects(const pt::ptree& ptree) { + std::vector objects{get_sources(ptree)}; + for (auto &i: objects) { + i.replace_extension("o"); + } + return objects; + } + + // both need to exist + bool is_older(const fs::path& p, const fs::path& other) { + auto t_p{fs::last_write_time(p)}; + auto t_other{fs::last_write_time(other)}; + return t_p < t_other; + } + + // outdated according to dependency file list, non-recursively + bool is_outdated(const fs::path& p, const std::vector &dependencies) { + if (!fs::exists(p)) + return true; + + for (const auto& dep: dependencies) { + if (!fs::exists(dep) || is_older(p, dep)) { + return true; + } + } + + return false; + } + + std::vector deps_from_depfile(const fs::path& path) { + std::string depfile_content{Reichwein::File::getFile(path)}; + std::vector parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")}; + if (parts.size() >= 2) { + std::vector deps {Reichwein::Stringhelper::split(parts[1], " ")}; + std::vector result; + std::transform(deps.cbegin(), deps.cend(), std::back_inserter(result), [](const std::string& s){ return s; }); + return result; + } else { + throw std::runtime_error("Bad depfile contents: "s + path.string()); + } + } + + fs::path depfile_name_from(const fs::path& p) { + fs::path depfile{p}; + depfile.replace_extension("d"); + return depfile; + } + + // return contained dependencies + // input: cpp + std::vector make_depfile_from(const fs::path& p) { + fs::path depfile{depfile_name_from(p)}; + + // check if depfile exists and if it contains up to date info + if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) { + // actually create depfile + int result{system(fmt::format("g++ -MM -MF {} -c {}", depfile.string(), p.string()).c_str())}; + if (result != 0) { + throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string())); + } + } + + return deps_from_depfile(depfile); + } + + // type of file can be built from dependencies + bool is_buildable_by_extension(const fs::path& p) { + fs::path ext{p.extension()}; + return ext.empty() || ext == ".o"; + } + + std::unordered_map> get_dependencies(const pt::ptree& ptree) { + std::unordered_map> dependencies; + dependencies.emplace(get_target(ptree), get_objects(ptree)); + std::vector sources{get_sources(ptree)}; + for (const auto& p: sources) { + fs::path p_obj{p}; + p_obj.replace_extension("o"); + std::vector deps {make_depfile_from(p)}; + // keep .d files for now to speed dependencies detection on following runs + //fs::remove(depfile_name_from(p)); + dependencies.emplace(p_obj, deps); + } + return dependencies; + } + +} + +Builder::Builder(const pt::ptree& ptree): + _target{get_target(ptree)}, + _objects{get_objects(ptree)}, + _sources{get_sources(ptree)}, + _dependencies{get_dependencies(ptree)} +{ +} + +std::vector Builder::dependencies_of(const fs::path& p) { + try { + return _dependencies.at(p); + } catch (const std::out_of_range& ex) { + return {}; // empty by default + } +} + +// outdated according to dependency tree, recursively +bool Builder::is_outdated(const fs::path& p) { + if (!fs::exists(p)) + return true; + + std::vector deps{dependencies_of(p)}; + for (const auto& dep: deps) { + if (!fs::exists(dep) || is_older(p, dep)) { + return true; + } + + if (is_outdated(dep)) { + return true; + } + } + + return false; +} + +// build 1 file +void Builder::build_file(const fs::path& p) { + std::string command; + + if (p.extension() == ".o") { + // compile + fs::path cppfile{p}; + cppfile.replace_extension("cpp"); + command = fmt::format("g++ -std=c++17 -c {} -o {}", cppfile.string(), p.string()); + } else { + // link + command = "g++"; + std::vector objects{dependencies_of(p)}; + for (auto &i: objects) { + command += fmt::format(" {}", i.string()); + } + command += " -lfmt -lreichwein"; + command += fmt::format(" -o {}", p.string()); + } + + std::cout << command << std::endl; + _runner.spawn(p.string(), command.c_str()); +} + +void Builder::cleanup() { + std::string path; + int exit_code{_runner.wait_one(path)}; + _activelist.erase(fs::path{path}); + + _donelist.insert(fs::path{path}); + if (exit_code != 0) { + throw std::runtime_error(fmt::format("Exit code {}", exit_code)); + } +} + +// build list of files +// eats up _buildlist +void Builder::build_list() { + + if (_buildlist.empty()) { + std::cout << "Everything up to date." << std::endl; + } else { + std::cout << "Running commands: " << std::endl; + } + + while (!_buildlist.empty()) { + // find file which can be built directly since its build dependencies are up to date + + fs::path current; + + while (current.empty()) { + for (auto &i: _buildlist) { + std::vector deps{dependencies_of(i)}; + bool deps_up_to_date{true}; + for (auto& dep: deps) { + if (_activelist.find(dep) != _activelist.end() || is_outdated(dep)) { + deps_up_to_date = false; + } + } + if (deps_up_to_date) { + current = i; + break; + } + } + + if (current.empty()) { + std::this_thread::sleep_for(10ms); // short wait before retry + if (_runner.finished() > 0) { + cleanup(); + } + } + } + + _buildlist.erase(current); + _activelist.insert(current); + + while (_runner.full() || _runner.finished() > 0) { + cleanup(); + } + + build_file(current); // calls spawn() on _runner + } + + // final cleanup + while (_runner.finished() != 0 || _runner.running() != 0) { + cleanup(); + } + + if (!_activelist.empty()) { + throw std::runtime_error("Files left actively building"); + } +} + +// build everything according to specified configuration +void Builder::build() { + std::cout << "Target: " << _target << std::endl; + + std::cout << "Sources: " << std::endl; + for (auto &i: _sources) { + std::cout << " " << i << std::endl; + } + + // create build list by depth-first search + std::cout << "Calculating build list..." << std::endl; + std::stack container; // temporary container for search algorithm + container.push(_target); + while (!container.empty()) { + fs::path current{container.top()}; + container.pop(); + + std::vector deps{dependencies_of(current)}; + for (auto &i: deps) { + container.push(i); + } + + if (is_outdated(current) && is_buildable_by_extension(current)) { + _buildlist.insert(current); + } + } + + std::cout << "Build list:" << std::endl; + for (auto &i: _buildlist) { + std::cout << " " << i << std::endl; + } + + build_list(); +} + +void Builder::clean() { + std::vector cleanlist{_objects}; + cleanlist.push_back(_target); + + std::vector commands; + for (auto &i: cleanlist) { + if (fs::exists(i)) { + commands.push_back(fmt::format("rm -f {}", i.string())); + } + fs::path depfile{depfile_name_from(i)}; + if (fs::exists(depfile)) { + commands.push_back(fmt::format("rm -f {}", depfile.string())); + } + } + + std::cout << "Running commands: " << std::endl; + for (auto &i: commands) { + std::cout << i << std::endl; + int result{system(i.c_str())}; + if (result != 0) { + throw std::runtime_error(fmt::format("Error {}", result)); + } + } +} + diff --git a/Builder.h b/Builder.h new file mode 100644 index 0000000..12338f0 --- /dev/null +++ b/Builder.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "ProcessRunner.h" + +class Builder +{ +public: + Builder(const boost::property_tree::ptree& ptree); + + void build(); + void clean(); + +private: + std::vector dependencies_of(const std::filesystem::path& p); + bool is_outdated(const std::filesystem::path& p); + + void build_file(const std::filesystem::path& p); + void build_list(); + + void cleanup(); + + std::filesystem::path _target; + std::vector _objects; + std::vector _sources; + std::unordered_map> _dependencies; + + ProcessRunner _runner; + + std::unordered_set _buildlist; // build done + std::unordered_set _activelist; // currently building + std::unordered_set _donelist; // build done +}; + diff --git a/Makefile b/Makefile index c24af4d..c0a8132 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROJECTNAME=ymake -SRC=main.cpp ymake.cpp builder.cpp +SRC=main.cpp ymake.cpp Builder.cpp ProcessRunner.cpp OBJ=$(SRC:.cpp=.o) diff --git a/ProcessRunner.cpp b/ProcessRunner.cpp new file mode 100644 index 0000000..bdde054 --- /dev/null +++ b/ProcessRunner.cpp @@ -0,0 +1,92 @@ +#include "ProcessRunner.h" + +#include +#include + +#include + +#include + +namespace bp = boost::process; + +ProcessRunner::ProcessRunner(): + _max_number_of_processes{static_cast(std::thread::hardware_concurrency())} +{} + +void ProcessRunner::spawn(const std::string& key, const std::string& command) +{ + _processes.emplace_back(process{key, command}); +} + +bool ProcessRunner::empty() +{ + return running() == 0; +} + +bool ProcessRunner::full() +{ + return running() == _max_number_of_processes; +} + +int ProcessRunner::wait_one(std::string& key) +{ + if (running() > 0 && finished() == 0) { + waitpid(-1, NULL, 0); + } + + // Actually, several childs may have finished. Therefore, end and + // remove all finished childs. + if (finished() > 0) { + for (auto it = _processes.begin(); it != _processes.end(); ++it) { + bp::child &child{it->child}; + if (!child.running()) { + child.wait(); + int result{child.exit_code()}; + + key = it->key; + + _processes.erase(it, it+1); + return result; + } + } + }; + + if (_processes.empty()) { + throw std::runtime_error("No process to wait for"); + } + + throw std::runtime_error("No process finished"); +} + +int ProcessRunner::wait_all() +{ + int ret{}; + for (auto &i: _processes) { + i.child.wait(); + int result{i.child.exit_code()}; + if (result != 0 && ret == 0) { + ret = result; + } + } + + _processes.clear(); + + return ret; +} + +size_t ProcessRunner::running() +{ + size_t result{}; + for (auto &i: _processes) { + if (i.child.running()) { + ++result; + } + } + return result; +} + +size_t ProcessRunner::finished() +{ + return _processes.size() - running(); +} + diff --git a/ProcessRunner.h b/ProcessRunner.h new file mode 100644 index 0000000..1c0789d --- /dev/null +++ b/ProcessRunner.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include + +struct process +{ + process(const std::string& k, const std::string& command): key{k}, child{command} {} + std::string key; + boost::process::child child; +}; + +// runs a number of processes in parallel, with an upper limit +class ProcessRunner +{ +public: + ProcessRunner(); + + void spawn(const std::string& key, const std::string& command); + + bool empty(); // number of running processes is zero + bool full(); // number of running processes equals maximum + + int wait_one(std::string& key); // returns exit code, and key via reference + int wait_all(); + + size_t running(); // number of running processes in queue + size_t finished(); // number of finished processes in queue + +private: + int _max_number_of_processes; + std::deque _processes; +}; + diff --git a/YMakefile b/YMakefile index 152ccc6..2881c10 100644 --- a/YMakefile +++ b/YMakefile @@ -1,8 +1,9 @@ ymake - builder.cpp + Builder.cpp main.cpp + ProcessRunner.cpp ymake.cpp diff --git a/builder.cpp b/builder.cpp deleted file mode 100644 index ee9c16b..0000000 --- a/builder.cpp +++ /dev/null @@ -1,296 +0,0 @@ -#include "builder.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include - -namespace fs = std::filesystem; -namespace bp = boost::process; -namespace pt = boost::property_tree; -using namespace std::string_literals; - -namespace { - fs::path get_target(const pt::ptree& ptree) { - return ptree.get("ymake.build.name"); - } - - std::vector get_sources(const pt::ptree& ptree) { - std::vector sources; - for (const pt::ptree::value_type &v: ptree.get_child("ymake.build")) { - if (v.first == "source") - sources.push_back(v.second.data()); - } - return sources; - } - - std::vector get_objects(const pt::ptree& ptree) { - std::vector objects{get_sources(ptree)}; - for (auto &i: objects) { - i.replace_extension("o"); - } - return objects; - } - - // both need to exist - bool is_older(const fs::path& p, const fs::path& other) { - auto t_p{fs::last_write_time(p)}; - auto t_other{fs::last_write_time(other)}; - return t_p < t_other; - } - - // outdated according to dependency file list, non-recursively - bool is_outdated(const fs::path& p, const std::vector &dependencies) { - if (!fs::exists(p)) - return true; - - for (const auto& dep: dependencies) { - if (!fs::exists(dep) || is_older(p, dep)) { - return true; - } - } - - return false; - } - - std::vector deps_from_depfile(const fs::path& path) { - std::string depfile_content{Reichwein::File::getFile(path)}; - std::vector parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")}; - if (parts.size() >= 2) { - std::vector deps {Reichwein::Stringhelper::split(parts[1], " ")}; - std::vector result; - std::transform(deps.cbegin(), deps.cend(), std::back_inserter(result), [](const std::string& s){ return s; }); - return result; - } else { - throw std::runtime_error("Bad depfile contents: "s + path.string()); - } - } - - fs::path depfile_name_from(const fs::path& p) { - fs::path depfile{p}; - depfile.replace_extension("d"); - return depfile; - } - - // return contained dependencies - // input: cpp - std::vector make_depfile_from(const fs::path& p) { - fs::path depfile{depfile_name_from(p)}; - - // check if depfile exists and if it contains up to date info - if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) { - // actually create depfile - int result{system(fmt::format("g++ -MM -MF {} -c {}", depfile.string(), p.string()).c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string())); - } - } - - return deps_from_depfile(depfile); - } - - // type of file can be built from dependencies - bool is_buildable_by_extension(const fs::path& p) { - fs::path ext{p.extension()}; - return ext.empty() || ext == ".o"; - } - - std::unordered_map> get_dependencies(const pt::ptree& ptree) { - std::unordered_map> dependencies; - dependencies.emplace(get_target(ptree), get_objects(ptree)); - std::vector sources{get_sources(ptree)}; - for (const auto& p: sources) { - fs::path p_obj{p}; - p_obj.replace_extension("o"); - std::vector deps {make_depfile_from(p)}; - // keep .d files for now to speed dependencies detection on following runs - //fs::remove(depfile_name_from(p)); - dependencies.emplace(p_obj, deps); - } - return dependencies; - } - -} - -Builder::Builder(const pt::ptree& ptree): - _target{get_target(ptree)}, - _objects{get_objects(ptree)}, - _sources{get_sources(ptree)}, - _dependencies{get_dependencies(ptree)} -{ -} - -std::vector Builder::dependencies_of(const fs::path& p) { - try { - return _dependencies.at(p); - } catch (const std::out_of_range& ex) { - return {}; // empty by default - } -} - -// outdated according to dependency tree, recursively -bool Builder::is_outdated(const fs::path& p) { - if (!fs::exists(p)) - return true; - - std::vector deps{dependencies_of(p)}; - for (const auto& dep: deps) { - if (!fs::exists(dep) || is_older(p, dep)) { - return true; - } - - if (is_outdated(dep)) { - return true; - } - } - - return false; -} - -// build 1 file -void Builder::build(const fs::path& p) { - std::string command; - - if (p.extension() == ".o") { - // compile - fs::path cppfile{p}; - cppfile.replace_extension("cpp"); - command = fmt::format("g++ -std=c++17 -c {} -o {}", cppfile.string(), p.string()); - } else { - // link - command = "g++"; - std::vector objects{dependencies_of(p)}; - for (auto &i: objects) { - command += fmt::format(" {}", i.string()); - } - command += " -lfmt -lreichwein"; - command += fmt::format(" -o {}", p.string()); - } - - std::cout << command << std::endl; - int result{system(command.c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Error {}", result)); - } -} - -// build list of files -// eats up input -void Builder::build(std::unordered_set& buildlist) { - std::unordered_set activelist; // currently building - std::unordered_set donelist; // build done - - if (buildlist.empty()) { - std::cout << "Everything up to date." << std::endl; - } else { - std::cout << "Running commands: " << std::endl; - } - - while (!buildlist.empty()) { - // find file which can be built directly since its build dependencies are up to date - // this holds for either any member of the set (e.g. at begin()), or any of its direct or indirect dependencies - fs::path current {*buildlist.begin()}; - - bool found_outdated{}; // outdated dependencies - - do { - found_outdated = false; - std::vector deps{dependencies_of(current)}; - for (auto& i: deps) { - if (activelist.find(i) == activelist.end() && is_outdated(i)) { - found_outdated = true; - current = i; - break; - } - } - } while (found_outdated); - - buildlist.erase(current); - - activelist.insert(current); - // TODO: build in background - build(current); - activelist.erase(current); - - donelist.insert(current); - } -} - -// build everything according to specified configuration -void Builder::build() { - std::cout << "Target: " << _target << std::endl; - - std::cout << "Sources: " << std::endl; - for (auto &i: _sources) { - std::cout << " " << i << std::endl; - } - - // create build list by depth-first search - std::cout << "Calculating build list..." << std::endl; - std::unordered_set buildlist; - std::stack container; // temporary container for search algorithm - container.push(_target); - while (!container.empty()) { - fs::path current{container.top()}; - container.pop(); - - std::vector deps{dependencies_of(current)}; - for (auto &i: deps) { - container.push(i); - } - - if (is_outdated(current) && is_buildable_by_extension(current)) { - buildlist.insert(current); - } - } - - std::cout << "Build list:" << std::endl; - for (auto &i: buildlist) { - std::cout << " " << i << std::endl; - } - - build(buildlist); -} - -void Builder::clean() { - std::vector cleanlist{_objects}; - cleanlist.push_back(_target); - - std::vector commands; - for (auto &i: cleanlist) { - if (fs::exists(i)) { - commands.push_back(fmt::format("rm -f {}", i.string())); - } - fs::path depfile{depfile_name_from(i)}; - if (fs::exists(depfile)) { - commands.push_back(fmt::format("rm -f {}", depfile.string())); - } - } - - std::cout << "Running commands: " << std::endl; - for (auto &i: commands) { - std::cout << i << std::endl; - int result{system(i.c_str())}; - if (result != 0) { - throw std::runtime_error(fmt::format("Error {}", result)); - } - } -} - diff --git a/builder.h b/builder.h deleted file mode 100644 index c139d2a..0000000 --- a/builder.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -class Builder -{ -public: - Builder(const boost::property_tree::ptree& ptree); - - void build(); - void clean(); - -private: - std::vector dependencies_of(const std::filesystem::path& p); - bool is_outdated(const std::filesystem::path& p); - - void build(const std::filesystem::path& p); - void build(std::unordered_set& buildlist); - - std::filesystem::path _target; - std::vector _objects; - std::vector _sources; - std::unordered_map> _dependencies; -}; diff --git a/ymake.cpp b/ymake.cpp index 93a813e..ad46d3c 100644 --- a/ymake.cpp +++ b/ymake.cpp @@ -1,6 +1,6 @@ #include "ymake.h" -#include "builder.h" +#include "Builder.h" #include #include -- cgit v1.2.3