Added stats output

10 files changed, 353 insertions, 49 deletions

diff --git a/Makefile b/Makefile
index 5b84f33..41c83c4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 DISTROS=debian10
 VERSION=$(shell dpkg-parsechangelog --show-field Version)
 PROJECTNAME=webserver
-PLUGINS=static-files webbox cgi weblog # fcgi
+PLUGINS=static-files webbox cgi weblog statistics # fcgi
 
 CXX=clang++-10
 
diff --git a/TODO b/TODO
index da9cf7c..d5dbf8b 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,4 @@
+Upload: read: body limit exceeded
 weblog: blättern
 weblog: link consistency check (cron?)
 weblog: style: zitate
@@ -5,7 +6,6 @@ Integrate into Debian
 Ubuntu version
 Speed up config.GetPath
 read: The socket was closed due to a timeout
-print statistics
 webbox: Info if not selected: all
 webbox: Copy function
 crypt pws
diff --git a/plugins/statistics/Makefile b/plugins/statistics/Makefile
new file mode 100644
index 0000000..48c2e8c
--- /dev/null
+++ b/plugins/statistics/Makefile
@@ -0,0 +1,124 @@
+DISTROS=debian10
+VERSION=$(shell dpkg-parsechangelog --show-field Version)
+PROJECTNAME=statistics
+
+CXX=clang++-10
+
+ifeq ($(shell which $(CXX)),)
+CXX=clang++
+endif
+
+ifeq ($(shell which $(CXX)),)
+CXX=g++-9
+endif
+
+ifeq ($(CXXFLAGS),)
+#CXXFLAGS=-O2 -DNDEBUG
+CXXFLAGS=-O0 -g -D_DEBUG
+endif
+# -fprofile-instr-generate -fcoverage-mapping
+# gcc:--coverage
+
+CXXFLAGS+= -Wall -I.
+
+CXXFLAGS+= -pthread -fvisibility=hidden -fPIC
+ifeq ($(CXX),clang++-10)
+CXXFLAGS+=-std=c++20 #-stdlib=libc++
+else
+CXXFLAGS+=-std=c++17
+endif
+
+CXXTESTFLAGS=-Igoogletest/include -Igooglemock/include/ -Igoogletest -Igooglemock
+
+LIBS=\
+-lboost_context \
+-lboost_coroutine \
+-lboost_program_options \
+-lboost_system \
+-lboost_thread \
+-lboost_filesystem \
+-lboost_regex \
+-lpthread \
+-lssl -lcrypto \
+-ldl
+
+ifeq ($(CXX),clang++-10)
+LIBS+= \
+-fuse-ld=lld-10 \
+-lstdc++
+#-lc++ \
+#-lc++abi
+#-lc++fs
+#-lstdc++fs
+else
+LIBS+= \
+-lstdc++ \
+-lstdc++fs
+endif
+
+PROGSRC=\
+    statistics.cpp
+
+TESTSRC=\
+    test-webserver.cpp \
+    googlemock/src/gmock-all.cpp \
+    googletest/src/gtest-all.cpp \
+    $(PROGSRC)
+
+SRC=$(PROGSRC)
+
+all: $(PROJECTNAME).so
+
+# testsuite ----------------------------------------------
+test-$(PROJECTNAME): $(TESTSRC:.cpp=.o)
+	$(CXX) $(CXXFLAGS) $^ $(LIBS) -o $@
+
+$(PROJECTNAME).so: $(SRC:.cpp=.o)
+	$(CXX) -shared $(CXXFLAGS) $^ $(LIBS) -o $@
+
+dep: $(TESTSRC:.cpp=.d)
+
+%.d: %.cpp
+	$(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -MM -MP -MF $@ -c $<
+
+%.o: %.cpp %.d
+	$(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@
+
+googletest/src/%.o: googletest/src/%.cc
+	$(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@
+
+# dependencies
+
+ADD_DEP=Makefile
+
+install:
+	mkdir -p $(DESTDIR)/usr/lib/webserver/plugins
+	cp $(PROJECTNAME).so $(DESTDIR)/usr/lib/webserver/plugins
+
+# misc ---------------------------------------------------
+deb:
+	# build binary deb package
+	dpkg-buildpackage -us -uc -rfakeroot
+
+deb-src:
+	dpkg-source -b .
+
+$(DISTROS): deb-src
+	sudo pbuilder build --basetgz /var/cache/pbuilder/$@.tgz --buildresult result/$@ ../webserver_$(VERSION).dsc ; \
+
+debs: $(DISTROS)
+
+clean:
+	-rm -f test-$(PROJECTNAME) $(PROJECTNAME)
+	-find . -name '*.o' -o -name '*.so' -o -name '*.d' -o -name '*.gcno' -o -name '*.gcda' | xargs rm -f
+
+zip: clean
+	-rm -f ../$(PROJECTNAME).zip
+	zip -r ../$(PROJECTNAME).zip *
+	ls -l ../$(PROJECTNAME).zip
+
+
+
+.PHONY: clean all zip install deb deb-src debs all $(DISTROS)
+
+-include $(wildcard $(SRC:.cpp=.d))
diff --git a/plugins/statistics/statistics.cpp b/plugins/statistics/statistics.cpp
new file mode 100644
index 0000000..03f4c94
--- /dev/null
+++ b/plugins/statistics/statistics.cpp
@@ -0,0 +1,97 @@
+#include "statistics.h"
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/coroutine2/coroutine.hpp>
+#include <boost/process.hpp>
+
+#include <algorithm>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+
+using namespace std::string_literals;
+namespace bp = boost::process;
+namespace fs = std::filesystem;
+
+namespace {
+
+ // Used to return errors by generating response page and HTTP status code
+ std::string HttpStatus(std::string status, std::string message, std::function<plugin_interface_setter_type>& SetResponseHeader)
+ {
+  SetResponseHeader("status", status);
+  SetResponseHeader("content_type", "text/html");
+  return status + " " + message;
+ }
+
+} // anonymous namespace
+
+std::string statistics_plugin::name()
+{
+ return "statistics";
+}
+
+statistics_plugin::statistics_plugin()
+{
+ //std::cout << "Plugin constructor" << std::endl;
+}
+
+statistics_plugin::~statistics_plugin()
+{
+ //std::cout << "Plugin destructor" << std::endl;
+}
+
+std::string statistics_plugin::generate_page(
+  std::function<std::string(const std::string& key)>& GetServerParam,
+  std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...)
+  std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string
+)
+{
+ try {
+  // Request path must not contain "..".
+  std::string rel_target{GetRequestParam("rel_target")};
+  size_t query_pos{rel_target.find("?")};
+  if (query_pos != rel_target.npos)
+   rel_target = rel_target.substr(0, query_pos);
+
+  std::string target{GetRequestParam("target")};
+  if (rel_target.find("..") != std::string::npos) {
+   return HttpStatus("400", "Illegal request: "s + target, SetResponseHeader);
+  }
+
+  // Build the path to the requested file
+  std::string doc_root{GetRequestParam("doc_root")};
+  fs::path path {fs::path{doc_root} / rel_target};
+  if (target.size() && target.back() != '/' && fs::is_directory(path)) {
+   std::string location{GetRequestParam("location") + "/"s};
+   SetResponseHeader("location", location);
+   return HttpStatus("301", "Correcting directory path", SetResponseHeader);
+  }
+
+  try {
+   SetResponseHeader("content_type", "text/html");
+
+   std::string header {"<!DOCTYPE html><html><head>"
+             "<meta charset=\"utf-8\"/>"
+             "<title>Webserver Statistics</title>"
+             "</head><body>"};
+   std::string footer{"<br/><br/><br/></body></html>"};
+
+   std::string result{header};
+
+   result += "<h1>Webserver Statistics</h1>";
+   result += "<pre>"s + GetServerParam("statistics") + "</pre>"s;
+
+   result += footer;
+   
+   return result;
+  } catch (const std::exception& ex) {
+   return HttpStatus("500", "Statistics error: "s + ex.what(), SetResponseHeader);
+  }
+
+ } catch (const std::exception& ex) {
+  return HttpStatus("500", "Unknown Error: "s + ex.what(), SetResponseHeader);
+ }
+}
+
diff --git a/plugins/statistics/statistics.h b/plugins/statistics/statistics.h
new file mode 100644
index 0000000..5db309b
--- /dev/null
+++ b/plugins/statistics/statistics.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "../../plugin_interface.h"
+
+class statistics_plugin: public webserver_plugin_interface 
+{
+public:
+ statistics_plugin();
+ ~statistics_plugin();
+ 
+ std::string name();
+ std::string generate_page(
+  std::function<std::string(const std::string& key)>& GetServerParam,
+  std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...)
+  std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string
+ );
+
+};
+
+extern "C" BOOST_SYMBOL_EXPORT statistics_plugin webserver_plugin;
+statistics_plugin webserver_plugin;
diff --git a/response.cpp b/response.cpp
index c5ba426..696b859 100644
--- a/response.cpp
+++ b/response.cpp
@@ -79,10 +79,11 @@ bool is_ipv6_address(const std::string& addr)
 
 std::unordered_map<std::string, std::function<std::string(Server&)>> GetServerParamFunctions{
  // following are the supported fields:
- {"version", [](Server& server) { return Server::VersionString; }},
  {"address", [](Server& server) { return server.GetSocket().address; }},
  {"ipv6", [](Server& server) { return is_ipv6_address(server.GetSocket().address) ? "yes" : "no"; }},
  {"port", [](Server& server) { return server.GetSocket().port; }},
+ {"statistics", [](Server& server) { return server.GetStatistics().getValues(); }},
+ {"version", [](Server& server) { return Server::VersionString; }},
 };
 
 std::string GetServerParam(const std::string& key, Server& server)
@@ -233,42 +234,34 @@ mime_type(beast::string_view path)
     return "application/text";
 }
 
-// Used to return errors by generating response page and HTTP status code
 response_type HttpStatus(std::string status, std::string message, response_type& res)
 {
- res.result(unsigned(stoul(status)));
- res.set(http::field::content_type, "text/html");
- if (res.result_int() == 401)
-  res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\"");
- res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s;
- res.prepare_payload();
+ if (status != "200") { // already handled at res init
+  res.result(unsigned(stoul(status)));
+  res.set(http::field::content_type, "text/html");
+  if (res.result_int() == 401)
+   res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\"");
+  res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s;
+  res.prepare_payload();
+ }
 
  return res;
 }
 
-// Do statistics at end of response generation, handle all exit paths via RAII
-class StatisticsGuard
+// Used to return errors by generating response page and HTTP status code
+response_type HttpStatusAndStats(std::string status, std::string message, RequestContext& req_ctx, response_type& res)
 {
- request_type& mReq;
- response_type& mRes;
- Server& mServer;
-public:
- StatisticsGuard(request_type& req, response_type& res, Server& server)
-  : mReq(req)
-  , mRes(res)
-  , mServer(server)
- {
- }
+ HttpStatus(status, message, res);
 
- ~StatisticsGuard()
- {
-  mServer.GetStatistics().count(mReq.body().size(),
-                                mRes.body().size(),
-                                mRes.result_int() == 200,
-                                is_ipv6_address(mServer.GetSocket().address),
-                                mServer.GetSocket().protocol == SocketProtocol::HTTPS);
- }
-};
+ req_ctx.GetServer().GetStatistics().count(
+   req_ctx.GetReq().body().size(),
+   res.body().size(),
+   res.result_int() != 200,
+   is_ipv6_address(req_ctx.GetServer().GetSocket().address),
+   req_ctx.GetServer().GetSocket().protocol == SocketProtocol::HTTPS);
+
+ return std::move(res);
+}
 
 } // anonymous namespace
 
@@ -279,8 +272,6 @@ response_type generate_response(request_type& req, Server& server)
  res.set(http::field::content_type, mime_type(extend_index_html(std::string(req.target()))));
  res.keep_alive(req.keep_alive());
 
- StatisticsGuard statsGuard{req, res, server};
-
  try {
   RequestContext req_ctx{req, server}; // can throw std::out_of_range
 
@@ -288,21 +279,21 @@ response_type generate_response(request_type& req, Server& server)
   if (auth.size() != 0) {
    std::string authorization{req[http::field::authorization]};
    if (authorization.substr(0, 6) != "Basic "s)
-    return HttpStatus("401", "Bad Authorization Type", res);
+    return HttpStatusAndStats("401", "Bad Authorization Type", req_ctx, res);
    
    authorization = authorization.substr(6);
    authorization = decode64(authorization);
 
    size_t pos {authorization.find(':')};
    if (pos == authorization.npos)
-    return HttpStatus("401", "Bad Authorization Encoding", res);
+    return HttpStatusAndStats("401", "Bad Authorization Encoding", req_ctx, res);
 
    std::string login{authorization.substr(0, pos)};
    std::string password{authorization.substr(pos + 1)};
 
    auto it {auth.find(login)};
    if (it == auth.end() || it->second != password)
-    return HttpStatus("401", "Bad Authorization", res);
+    return HttpStatusAndStats("401", "Bad Authorization", req_ctx, res);
   }
 
   plugin_type plugin{req_ctx.GetPlugin()};
@@ -318,8 +309,8 @@ response_type generate_response(request_type& req, Server& server)
    res.body() = res_data;
    res.prepare_payload();
   }
-
-  return res;
+  
+  return HttpStatusAndStats("200", "OK", req_ctx, res);
  } catch(const std::out_of_range& ex) {
   return HttpStatus("400", "Bad request: Host "s + std::string{req["host"]} + ":"s + std::string{req.target()} + " unknown"s, res);
  } catch(const std::exception& ex) {
diff --git a/server.cpp b/server.cpp
index 71f39ac..5d1609d 100644
--- a/server.cpp
+++ b/server.cpp
@@ -20,6 +20,7 @@
 #include <boost/config.hpp>
 
 #include <exception>
+#include <functional>
 #include <iostream>
 #include <thread>
 #include <vector>
@@ -39,6 +40,10 @@ using tcp = boost::asio::ip::tcp;       // from <boost/asio/ip/tcp.hpp>
 
 const std::string Server::VersionString{ "Reichwein.IT Webserver "s + std::string{VERSION} };
 
+namespace {
+ const int32_t stats_timer_seconds { 24 * 60 * 60 }; // save stats once a day
+} // anonymous namespace
+
 Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics)
  : m_config(config)
  , m_ioc(ioc)
@@ -66,6 +71,16 @@ int run_server(Config& config, plugins_container_type& plugins)
                     ioc.stop();
                     });
 
+ // Save stats once a day
+ boost::asio::steady_timer stats_save_timer(ioc, boost::asio::chrono::seconds(stats_timer_seconds));
+ std::function<void(const boost::system::error_code&)> stats_callback =
+   [&](const boost::system::error_code& error){
+         stats.save();
+         stats_save_timer.expires_at(stats_save_timer.expires_at() + boost::asio::chrono::seconds(stats_timer_seconds));
+         stats_save_timer.async_wait(stats_callback);
+   };
+ stats_save_timer.async_wait(stats_callback);
+
  std::vector<std::shared_ptr<Server>> servers;
 
  const auto& sockets {config.Sockets()};
diff --git a/statistics.cpp b/statistics.cpp
index 19d0258..3fb99a3 100644
--- a/statistics.cpp
+++ b/statistics.cpp
@@ -5,13 +5,15 @@
 #include <iostream>
 
 namespace fs = std::filesystem;
+using namespace std::string_literals;
 
 namespace {
  const fs::path statsfilepath{ "/var/lib/webserver/stats.db" };
 } // anonymous namespace
 
-Statistics::Statistics()
+void Statistics::load()
 {
+ std::lock_guard<std::mutex> lock(mMutex);
  std::cout << "Loading statistics..." << std::endl;
  std::ifstream file{statsfilepath, std::ios::in | std::ios::binary};
  if (file.is_open()) {
@@ -21,22 +23,38 @@ Statistics::Statistics()
  } else {
   std::cerr << "Warning: Couldn't read statistics" << std::endl;
  }
+
+ mChanged = false;
 }
 
-Statistics::~Statistics()
+void Statistics::save()
 {
- std::cout << "Saving statistics..." << std::endl;
- std::lock_guard<std::mutex> lock(mMutex);
- std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc};
- if (file.is_open()) {
-  Serialization::OArchive archive{file};
-
-  archive << mBins;
- } else {
-  std::cerr << "Warning: Couldn't write statistics" << std::endl;
+ if (mChanged) {
+  std::lock_guard<std::mutex> lock(mMutex);
+  std::cout << "Saving statistics..." << std::endl;
+  std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc};
+  if (file.is_open()) {
+   Serialization::OArchive archive{file};
+
+   archive << mBins;
+  } else {
+   std::cerr << "Warning: Couldn't write statistics" << std::endl;
+  }
+
+  mChanged = false;
  }
 }
 
+Statistics::Statistics()
+{
+ load();
+}
+
+Statistics::~Statistics()
+{
+ save();
+}
+
 bool Statistics::Bin::expired() const
 {
  auto now {time(nullptr)};
@@ -57,6 +75,8 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6,
 {
  std::lock_guard<std::mutex> lock(mMutex);
 
+ mChanged = true;
+
  if (mBins.empty() || mBins.back().expired()) {
   mBins.emplace_back(Bin{static_cast<uint64_t>((time(nullptr) / binsize) * binsize), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
  }
@@ -85,3 +105,30 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6,
  limit();
 }
 
+std::string Statistics::getValues()
+{
+ std::lock_guard<std::mutex> lock(mMutex);
+
+ std::string result;
+
+ for (const auto& bin: mBins) {
+  result += std::to_string(bin.start_time) + ","s +
+
+            std::to_string(bin.requests) + ","s +
+            std::to_string(bin.errors) + ","s +
+            std::to_string(bin.bytes_in) + ","s +
+            std::to_string(bin.bytes_out) + ","s +
+
+            std::to_string(bin.requests_ipv6) + ","s +
+            std::to_string(bin.errors_ipv6) + ","s +
+            std::to_string(bin.bytes_in_ipv6) + ","s +
+            std::to_string(bin.bytes_out_ipv6) + ","s +
+
+            std::to_string(bin.requests_https) + ","s +
+            std::to_string(bin.errors_https) + ","s +
+            std::to_string(bin.bytes_in_https) + ","s +
+            std::to_string(bin.bytes_out_https) + "\n"s;
+ }
+
+ return result;
+}
diff --git a/statistics.h b/statistics.h
index c4fce93..7e4da7e 100644
--- a/statistics.h
+++ b/statistics.h
@@ -60,9 +60,11 @@ public:
  };
 
 private:
+ bool mChanged{};
  std::deque<Bin> mBins;
  std::mutex mMutex;
 
+ void load();
  void limit();
 
 public:
@@ -70,6 +72,9 @@ public:
  ~Statistics();
 
  void count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https);
+ void save();
+
+ std::string getValues();
 };
 
 // Serialization and Deserialization as free functions
diff --git a/webserver.conf b/webserver.conf
index 52075a2..5adc9ba 100644
--- a/webserver.conf
+++ b/webserver.conf
@@ -38,6 +38,10 @@
     <WEBLOG_DESCRIPTION>Roland Reichweins Blog</WEBLOG_DESCRIPTION>
     <WEBLOG_KEYWORDS>Roland Reichwein, Blog</WEBLOG_KEYWORDS>
    </path>
+   <path requested="/stats">
+    <plugin>statistics</plugin>
+    <target></target>
+   </path>
    <path requested="/cgi-bin">
     <plugin>cgi</plugin>
     <target>/home/ernie/code/webserver/cgi-bin</target>