path: root/Builder.cpp

#include "Builder.h"

#include "file.h"

#include <algorithm>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <iterator>
#include <sstream>
#include <stack>
#include <stdexcept>
#include <string>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include <boost/process.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>

#include <fmt/format.h>

#include <libreichwein/file.h>
#include <libreichwein/stringhelper.h>

namespace bp = boost::process;
namespace fs = std::filesystem;
namespace pt = boost::property_tree;
using namespace std::chrono_literals;
using namespace std::string_literals;

namespace {
 const std::string topelement{"ymake"};

 bool is_match(const pt::ptree::value_type& element, const std::string& target) {
  if (target == "*") {
   return true;
  } else {
   return element.first == target;
  }
 }

 // get name of single target, ptree is <build> subtree
 fs::path get_target(const pt::ptree& ptree) {
  return ptree.get<std::string>("name");
 }

 // ptree is main tree
 std::vector<fs::path> get_all_targets(const pt::ptree& ptree, const std::string& target) {
  std::vector<fs::path> result;

  // iterate over all <build> elements
  for (const auto& element: ptree.get_child(topelement)) {
   if (is_match(element, target)) {
    result.push_back(element.second.get<std::string>("name"));
   }
  }

  return result;
 }

 // get sources of single target, ptree is <build> subtree
 std::vector<fs::path> get_sources(const pt::ptree& ptree) {
  std::vector<fs::path> sources;
  for (const pt::ptree::value_type &v: ptree) {
   if (v.first == "source")
    sources.push_back(v.second.data());
  }
  return sources;
 }

 // ptree is main tree
 std::vector<fs::path> get_all_sources(const pt::ptree& ptree, const std::string& target) {
  std::vector<fs::path> result;

  // iterate over all <build> elements
  for (const auto& element: ptree.get_child(topelement)) {
   if (is_match(element, target)) {
    // iterate over all <source> elements
    for (const auto& source: element.second) {
     if (source.first == "source") {
      result.push_back(source.second.data());
     }
    }
   }
  }

  return result;
 }

 // get objects for corresponding sources of single target,
 // ptree is <build> subtree
 std::vector<fs::path> get_objects(const pt::ptree& ptree) {
  std::vector<fs::path> objects{get_sources(ptree)};
  for (auto &i: objects) {
   i.replace_extension("o");
  }
  return objects;
 }

 // ptree is main tree
 std::vector<fs::path> get_all_objects(const pt::ptree& ptree, const std::string& target) {
  std::vector<fs::path> objects{get_all_sources(ptree, target)};
  for (auto &i: objects) {
   i.replace_extension("o");
  }
  return objects;
 }

 // both need to exist
 bool is_older(const fs::path& p, const fs::path& other) {
  auto t_p{fs::last_write_time(p)};
  auto t_other{fs::last_write_time(other)};
  return t_p < t_other;
 }

 std::vector<fs::path> deps_from_depfile(const fs::path& path) {
  std::string depfile_content{Reichwein::File::getFile(path)};
  std::vector<std::string> parts {Reichwein::Stringhelper::split(depfile_content, ":\r\n")};
  if (parts.size() >= 2) {
   std::vector<std::string> deps {Reichwein::Stringhelper::split(parts[1], " ")};
   std::vector<fs::path> result;
   std::copy(deps.cbegin(), deps.cend(), std::back_inserter(result));
   return result;
  } else {
   throw std::runtime_error("Bad depfile contents: "s + path.string());
  }
 }

 fs::path depfile_name_from(const fs::path& p) {
  fs::path depfile{p};
  depfile.replace_extension("d");
  return depfile;
 }

 // T = std::string, fs::path
 template<typename T>
 // get specified subelements (as list) of single target, ptree is <build> subtree
 std::vector<T> get_subelements_of_build(const pt::ptree& ptree, const std::string& subelement) {
  std::vector<T> result;
  for (const pt::ptree::value_type &v: ptree) {
   if (v.first == subelement) {
    result.push_back(v.second.data());
   }
  }
  return result;
 }

 // T = std::string, fs::path
 template<typename T>
 std::unordered_map<fs::path, std::vector<T>> get_subelements(const pt::ptree& ptree, const std::string& target, const std::string& subelement) {
  std::unordered_map<fs::path, std::vector<T>> result_map;

  for (const auto& element: ptree.get_child(topelement)) {
   if (is_match(element, target)) {
    std::vector<T> subelements{get_subelements_of_build<T>(element.second, subelement)};
    if (!subelements.empty()) {
     result_map.emplace(get_target(element.second), subelements);
    }
   }
  }

  return result_map;
 }
}

Builder::Builder(const pt::ptree& ptree, const std::string& target):
 _ptree(ptree),
 _target(target),
 _all_targets{get_all_targets(ptree, target)},
 _all_objects{get_all_objects(ptree, target)},
 _include_paths{get_subelements<fs::path>(ptree, target, "includepath")},
 _link_libs{get_subelements<std::string>(ptree, target, "linklib")}
{
 // intentionally defer creation of _dependencies to build()
 // to prevent creation of .d files in clean()
}

// given a compiled compile unit (*.o), find its source file
fs::path Builder::get_compile_unit_source_from_object(const fs::path& path) {
 std::vector<fs::path> source_files{dependencies_of(path)};
 auto it{std::find_if(source_files.begin(), source_files.end(), is_compile_unit_source_by_extension)};
 if (it == source_files.end()) {
  throw std::runtime_error(fmt::format("No source file found for {}", path.string()));
 }
 return *it;
}

std::vector<fs::path> Builder::dependencies_of(const fs::path& p) const
{
 try {
  return _dependencies.at(p);
 } catch (const std::out_of_range& ex) {
  return {}; // empty by default
 }
}

std::vector<std::string> Builder::link_libs_of(const fs::path& p) const
{
 try {
  return _link_libs.at(p);
 } catch (const std::out_of_range& ex) {
  return {}; // empty by default
 }
}

fs::path Builder::target_from_object(const fs::path& object) const
{
 for (const auto& target: _all_targets) {
  const auto& deps{dependencies_of(target)};
  auto it{std::find(deps.cbegin(), deps.cend(), object)};
  if (it != deps.cend()) {
   return target;
  }
 }

 throw std::runtime_error("No target found which depends on object "s + object.string());
}

std::vector<fs::path> Builder::include_paths_of_object(const fs::path& object) const
{
 fs::path target{target_from_object(object)};

 // include paths from target
 try {
  return _include_paths.at(target);
 } catch (const std::out_of_range& ex) {
  return {}; // empty by default
 }
}

// outdated according to dependency tree, recursively
bool Builder::is_outdated(const fs::path& p) const
{
 if (!fs::exists(p))
  return true;

 std::vector<fs::path> deps{dependencies_of(p)};
 for (const auto& dep: deps) {
  if (!fs::exists(dep) || is_older(p, dep)) {
   return true;
  }

  if (is_outdated(dep)) {
   return true;
  }
 }

 return false;
}

// outdated according to dependency file list, non-recursively
bool Builder::is_outdated(const fs::path& p, const std::vector<fs::path> &dependencies) const
{
 if (!fs::exists(p))
  return true;

 for (const auto& dep: dependencies) {
  if (!fs::exists(dep) || is_older(p, dep)) {
   return true;
  }
 }

 return false;
}

// return contained dependencies
// input: cpp
std::vector<fs::path> Builder::make_depfile_from(const fs::path& p) const
{
 fs::path depfile{depfile_name_from(p)};

 // check if depfile exists and if it contains up to date info
 if (!fs::exists(depfile) || is_outdated(depfile, deps_from_depfile(depfile))) {
  // actually create depfile
  int result{system(_lang.getDepCommand(depfile.string(), p.string()).c_str())};
  if (result != 0) {
   throw std::runtime_error(fmt::format("Depfile {} can't be created", depfile.string()));
  }
 }

 return deps_from_depfile(depfile);
}

std::unordered_map<fs::path, std::vector<fs::path>> Builder::get_dependencies(const pt::ptree& ptree) const
{
 std::unordered_map<fs::path, std::vector<fs::path>> dependencies;

 for (const auto& element: ptree.get_child(topelement)) {
  if (is_match(element, _target)) {
   dependencies.emplace(get_target(element.second), get_objects(element.second));

   std::vector<fs::path> sources{get_sources(element.second)};
   for (const auto& p: sources) {
    fs::path p_obj{p};
    p_obj.replace_extension("o");
    std::vector<fs::path> deps {make_depfile_from(p)};
    // keep .d files for now to speed dependencies detection on following runs
    //fs::remove(depfile_name_from(p));
    dependencies.emplace(p_obj, deps);
   }
  }
 }

 return dependencies;
}

// build 1 file
void Builder::build_file(const fs::path& p) {
 std::string command;

 if (p.extension() == ".o") {
  // compile
  fs::path source_file{get_compile_unit_source_from_object(p)};
  command = _lang.getCompileCommand(p, source_file, include_paths_of_object(p));
 } else {
  // link
  std::vector<fs::path> objects{dependencies_of(p)};
  std::vector<std::string> link_libs{link_libs_of(p)};
  command = _lang.getLinkCommand(p, objects, link_libs);
 }

 std::cout << command << std::endl;
 _runner.spawn(p.string(), command.c_str());
}

void Builder::cleanup_buildlist() {
 std::string path;
 int exit_code{_runner.wait_one(path)};
 _activelist.erase(fs::path{path});

 _donelist.insert(fs::path{path});
 if (exit_code != 0) {
  throw std::runtime_error(fmt::format("Exit code {}", exit_code));
 }
}

// build list of files
// eats up _buildlist
void Builder::build_filelist() {

 if (_buildlist.empty()) {
  std::cout << "Everything up to date." << std::endl;
 } else {
  // std::cout << "Running commands: " << std::endl;
 }

 while (!_buildlist.empty()) {
  // find file which can be built directly since its build dependencies are up to date

  fs::path current;

  while (current.empty()) {
   for (auto &i: _buildlist) {
    std::vector<fs::path> deps{dependencies_of(i)};
    bool deps_up_to_date{true};
    for (auto& dep: deps) {
     if (_activelist.find(dep) != _activelist.end() || is_outdated(dep)) {
      deps_up_to_date = false;
     }
    }
    if (deps_up_to_date) {
     current = i;
     break;
    }
   }

   if (current.empty()) {
    std::this_thread::sleep_for(10ms); // short wait before retry
    if (_runner.finished() > 0) {
     cleanup_buildlist();
    }
   }
  }

  _buildlist.erase(current);
  _activelist.insert(current);

  // wait until process slot is available
  while (_runner.full() || _runner.finished() > 0) {
   cleanup_buildlist();
  }

  build_file(current); // calls spawn() on _runner
 }

 // final cleanup
 while (_runner.finished() != 0 || _runner.running() != 0) {
  cleanup_buildlist();
 }

 if (!_activelist.empty()) {
  throw std::runtime_error("Files left actively building");
 }
}

// build everything according to specified configuration
void Builder::build() {
 _dependencies = get_dependencies(_ptree);

 // create build list by depth-first search
 //std::cout << "Calculating build list..." << std::endl;
 std::stack<fs::path> container; // temporary container for search algorithm

 for (const auto& target: _all_targets) {
  container.push(target);
 }

 while (!container.empty()) {
  fs::path current{container.top()};
  container.pop();

  std::vector<fs::path> deps{dependencies_of(current)};
  for (auto &i: deps) {
   container.push(i);
  }

  if (is_outdated(current) && is_buildable_by_extension(current)) {
   _buildlist.insert(current);
  }
 }

 //std::cout << "Build list:" << std::endl;
 //for (auto &i: _buildlist) {
 // std::cout << "  " << i << std::endl;
 //}

 build_filelist();
}

void Builder::clean() const {
 std::vector<fs::path> cleanlist{_all_objects};
 std::copy(_all_targets.cbegin(), _all_targets.cend(), std::back_inserter(cleanlist));

 std::vector<std::string> commands;
 for (auto &i: cleanlist) {
  if (fs::exists(i)) {
   commands.push_back(fmt::format("rm -f {}", i.string()));
  }
  if (i.extension() == ".o") {
   fs::path depfile{depfile_name_from(i)};
   if (fs::exists(depfile)) {
    commands.push_back(fmt::format("rm -f {}", depfile.string()));
   }
  }
 }

 //std::cout << "Running commands: " << std::endl;
 for (auto &i: commands) {
  std::cout << i << std::endl;
  int result{system(i.c_str())};
  if (result != 0) {
   throw std::runtime_error(fmt::format("Error {}", result));
  }
 }
}

// build tests and run them
void Builder::run_tests() {
 for (const auto& i: _all_targets) {
  std::string command{(fs::path("./") / i).string()};
  std::cout << "Running test: " << command << std::endl;
  int result {bp::system(command)};
  if (result != 0) {
   throw std::runtime_error("Test failed: " + i.string() + ", exit code: " + std::to_string(result));
  }
 }
}