Added statistics

14 files changed, 473 insertions, 20 deletions

diff --git a/Makefile b/Makefile
index de0ce07..5b84f33 100644
--- a/Makefile
+++ b/Makefile
@@ -67,6 +67,7 @@ PROGSRC=\
     plugin.cpp \
     privileges.cpp \
     response.cpp \
+    statistics.cpp \
     server.cpp
 
 TESTSRC=\
diff --git a/TODO b/TODO
index b8a8bce..6d7ac91 100644
--- a/TODO
+++ b/TODO
@@ -2,6 +2,7 @@ weblog: blättern
 weblog: link consistency check (cron?)
 weblog: style: zitate
 Integrate into Debian
+debian: restart server on install/update
 Ubuntu version
 Speed up config.GetPath
 read: The socket was closed due to a timeout
diff --git a/archive.h b/archive.h
new file mode 100644
index 0000000..a98921a
--- /dev/null
+++ b/archive.h
@@ -0,0 +1,206 @@
+#pragma once
+
+#include <boost/coroutine2/coroutine.hpp>
+#include <boost/endian/conversion.hpp>
+
+#include <cstdint>
+#include <ostream>
+#include <istream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+typedef boost::coroutines2::coroutine<void> coro_t;
+
+// Serialization, similar to Boost Serialization
+// but for portable binary archive
+// using big endian coding (network byte order)
+namespace Serialization {
+
+class OArchive
+{
+public:
+ OArchive(std::ostream& os): os(os) {}
+ ~OArchive() {}
+
+ template<class T>
+ OArchive& operator &(T& v) {
+  v.serialize(*this);
+
+  return *this;
+ };
+
+ template <class T>
+ OArchive& write_fundamental(T& v)
+ {
+  T value = boost::endian::native_to_big(v);
+  os.write((char*)&value, sizeof(value));
+  return *this;
+ }
+
+ OArchive& operator &(uint8_t& v)
+ {
+  return write_fundamental(v);
+ };
+
+ OArchive& operator &(uint16_t& v)
+ {
+  return write_fundamental(v);
+ };
+
+ OArchive& operator &(uint32_t& v)
+ {
+  return write_fundamental(v);
+ };
+
+ OArchive& operator &(uint64_t& v)
+ {
+  return write_fundamental(v);
+ };
+
+ OArchive& operator &(int64_t& v)
+ {
+  return write_fundamental(*reinterpret_cast<uint64_t*>(v));
+ };
+
+ OArchive& operator &(std::vector<uint8_t>& v)
+ {
+  uint32_t size = static_cast<uint32_t>(v.size());
+  *this & size;
+  os.write((char*)v.data(), size);
+  return *this;
+ };
+
+ OArchive& operator &(std::string& v)
+ {
+  uint32_t size = static_cast<uint32_t>(v.size());
+  *this & size;
+  os.write((char*)v.data(), v.size());
+  return *this;
+ };
+
+private:
+ std::ostream &os;
+};
+
+class IArchive
+{
+public:
+ IArchive(std::istream& is): is(is) {}
+ IArchive(std::stringstream& is, coro_t::pull_type& coro) : is(is), mStringStream(&is), mCoro(&coro) {}
+ ~IArchive() {}
+
+ template<class T>
+ IArchive& operator &(T& v)
+ {
+  v.serialize(*this);
+
+  return *this;
+ };
+
+ template <class T>
+ IArchive& read_fundamental(T& v)
+ {
+  // in coroutine case, wait for input, if necessary
+  if (mCoro && mStringStream) {
+   while (mStringStream->tellp() - mStringStream->tellg() < sizeof(v)) {
+    (*mCoro)();
+   }
+  }
+
+  // now, we have enough bytes available
+  T value;
+  is.read((char*)&value, sizeof(value));
+  v = boost::endian::big_to_native(value);
+  return *this;
+ }
+
+ IArchive& operator &(uint8_t& v)
+ {
+  return read_fundamental(v);
+ };
+
+ IArchive& operator &(uint16_t& v)
+ {
+  return read_fundamental(v);
+ };
+
+ IArchive& operator &(uint32_t& v)
+ {
+  return read_fundamental(v);
+ };
+
+ IArchive& operator &(uint64_t& v)
+ {
+  return read_fundamental(v);
+ };
+
+ IArchive& operator &(int64_t& v)
+ {
+  uint64_t uv;
+  read_fundamental(uv);
+  v = *reinterpret_cast<int64_t*>(uv);
+  return *this;
+ };
+
+ template <class T>
+ IArchive& read_bytes_vector(T& v)
+ {
+  uint32_t size;
+  *this & size;
+
+  v.resize(size);
+
+  // in coroutine case, wait for input, if necessary
+  if (mCoro && mStringStream) {
+   while (mStringStream->tellp() - mStringStream->tellg() < size) {
+    (*mCoro)();
+   }
+  }
+
+  // now, we have enough bytes available
+  is.read((char*)v.data(), size);
+  return *this;
+ }
+
+ IArchive& operator &(std::vector<uint8_t>& v)
+ {
+  return read_bytes_vector(v);
+ };
+
+ IArchive& operator &(std::string& v)
+ {
+  return read_bytes_vector(v);
+ };
+
+private:
+ std::istream &is;
+ std::stringstream* mStringStream{ }; // for i/o sizes access
+ coro_t::pull_type* mCoro{ }; // optional for coroutine
+};
+
+// - Free functions ----------------------------------------------------------
+
+template<class Archive, class T>
+void serialize(Archive& ar, T& v)
+{
+ ar & v;
+}
+
+template<class T>
+OArchive& operator <<(OArchive& ar, T& v)
+{
+ serialize(ar, v);
+
+ return ar;
+};
+
+template<class T>
+IArchive& operator >>(IArchive& ar, T& v)
+{
+ serialize(ar, v);
+
+ return ar;
+};
+
+}
diff --git a/debian/changelog b/debian/changelog
index dc8df04..5b7f88a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 webserver (1.3) unstable; urgency=medium
 
   * Updated weblog
+  * Added statistics
 
  -- Roland Reichwein <rr@antcom.de>  Sat, 25 Apr 2020 12:35:44 +0200
 
diff --git a/debian/webserver.dirs b/debian/webserver.dirs
new file mode 100644
index 0000000..23315d6
--- /dev/null
+++ b/debian/webserver.dirs
@@ -0,0 +1 @@
+var/lib/webserver
diff --git a/http.cpp b/http.cpp
index cb95b0f..a4709bb 100644
--- a/http.cpp
+++ b/http.cpp
@@ -357,8 +357,8 @@ private:
 
 namespace HTTP {
 
- Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins)
-  : ::Server(config, ioc, socket, plugins)
+ Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics)
+  : ::Server(config, ioc, socket, plugins, statistics)
  {
  }
 
diff --git a/http.h b/http.h
index c8f27a3..7c3bb9a 100644
--- a/http.h
+++ b/http.h
@@ -11,7 +11,7 @@ namespace HTTP {
 class Server: public ::Server
 {
 public:
- Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins);
+ Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics);
  virtual ~Server();
  int start() override;
 };
diff --git a/https.cpp b/https.cpp
index 5230d60..7c94099 100644
--- a/https.cpp
+++ b/https.cpp
@@ -630,8 +630,8 @@ int servername_callback(SSL *s, int *al, void *arg)
 
 namespace HTTPS {
 
-Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins)
- : ::Server(config, ioc, socket, plugins)
+Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics)
+ : ::Server(config, ioc, socket, plugins, statistics)
 {
  // initial dummy, before we can add specific ctx w/ certificate
  std::shared_ptr<ssl::context> ctx_dummy{std::make_shared<ssl::context>(tls_method)};
diff --git a/https.h b/https.h
index 864f379..738e122 100644
--- a/https.h
+++ b/https.h
@@ -40,7 +40,7 @@ private:
  ctx_type m_ctx;
  
 public:
- Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins);
+ Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics);
  virtual ~Server();
 
  int start() override;
diff --git a/response.cpp b/response.cpp
index e1b6c05..c5ba426 100644
--- a/response.cpp
+++ b/response.cpp
@@ -246,6 +246,30 @@ response_type HttpStatus(std::string status, std::string message, response_type&
  return res;
 }
 
+// Do statistics at end of response generation, handle all exit paths via RAII
+class StatisticsGuard
+{
+ request_type& mReq;
+ response_type& mRes;
+ Server& mServer;
+public:
+ StatisticsGuard(request_type& req, response_type& res, Server& server)
+  : mReq(req)
+  , mRes(res)
+  , mServer(server)
+ {
+ }
+
+ ~StatisticsGuard()
+ {
+  mServer.GetStatistics().count(mReq.body().size(),
+                                mRes.body().size(),
+                                mRes.result_int() == 200,
+                                is_ipv6_address(mServer.GetSocket().address),
+                                mServer.GetSocket().protocol == SocketProtocol::HTTPS);
+ }
+};
+
 } // anonymous namespace
 
 response_type generate_response(request_type& req, Server& server)
@@ -255,6 +279,8 @@ response_type generate_response(request_type& req, Server& server)
  res.set(http::field::content_type, mime_type(extend_index_html(std::string(req.target()))));
  res.keep_alive(req.keep_alive());
 
+ StatisticsGuard statsGuard{req, res, server};
+
  try {
   RequestContext req_ctx{req, server}; // can throw std::out_of_range
 
@@ -275,11 +301,8 @@ response_type generate_response(request_type& req, Server& server)
    std::string password{authorization.substr(pos + 1)};
 
    auto it {auth.find(login)};
-   if (it == auth.end())
+   if (it == auth.end() || it->second != password)
     return HttpStatus("401", "Bad Authorization", res);
-
-   if (it->second != password)
-    return HttpStatus("401", "Bad Authorization", res); // should be same message as previous one to prevent login guessing
   }
 
   plugin_type plugin{req_ctx.GetPlugin()};
diff --git a/server.cpp b/server.cpp
index 6e15466..71f39ac 100644
--- a/server.cpp
+++ b/server.cpp
@@ -15,6 +15,7 @@
 #include <boost/asio/ssl/stream.hpp>
 #endif
 #include <boost/asio/dispatch.hpp>
+#include <boost/asio/signal_set.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/config.hpp>
 
@@ -28,6 +29,7 @@
 #include "http.h"
 #include "https.h"
 #include "privileges.h"
+#include "statistics.h"
 
 namespace beast = boost::beast;         // from <boost/beast.hpp>
 namespace http = beast::http;           // from <boost/beast/http.hpp>
@@ -37,11 +39,12 @@ using tcp = boost::asio::ip::tcp;       // from <boost/asio/ip/tcp.hpp>
 
 const std::string Server::VersionString{ "Reichwein.IT Webserver "s + std::string{VERSION} };
 
-Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins)
+Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics)
  : m_config(config)
  , m_ioc(ioc)
  , m_socket(socket)
  , m_plugins(plugins)
+ , m_statistics(statistics)
 {
 }
 
@@ -51,18 +54,26 @@ Server::~Server()
 
 int run_server(Config& config, plugins_container_type& plugins)
 {
+ Statistics stats;
+
  auto const threads = std::max<int>(1, config.Threads());
 
  boost::asio::io_context ioc{threads};
 
+ boost::asio::signal_set signals(ioc, SIGINT, SIGTERM);
+ signals.async_wait([&](const boost::system::error_code& error, int signal_number){
+                    std::cout << "Terminating via signal " << signal_number << std::endl;
+                    ioc.stop();
+                    });
+
  std::vector<std::shared_ptr<Server>> servers;
 
  const auto& sockets {config.Sockets()};
  for (const auto& socket: sockets) {
   if (socket.protocol == SocketProtocol::HTTP) {
-   servers.push_back(std::make_shared<HTTP::Server>(config, ioc, socket, plugins));
+   servers.push_back(std::make_shared<HTTP::Server>(config, ioc, socket, plugins, stats));
   } else {
-   servers.push_back(std::make_shared<HTTPS::Server>(config, ioc, socket, plugins));
+   servers.push_back(std::make_shared<HTTPS::Server>(config, ioc, socket, plugins, stats));
   }
   servers.back()->start();
  }
@@ -73,14 +84,19 @@ int run_server(Config& config, plugins_container_type& plugins)
  // Run the I/O service on the requested number of threads
  std::vector<std::thread> v;
  v.reserve(threads - 1);
- for(auto i = threads - 1; i > 0; --i)
-     v.emplace_back(
-     [&ioc]
-     {
-         ioc.run();
-     });
+ for (auto i = threads - 1; i > 0; --i) {
+  v.emplace_back(
+  [&ioc]
+  {
+   ioc.run();
+  });
+ }
  ioc.run();
 
+ for (auto& t: v) {
+  t.join();
+ }
+
  return EXIT_SUCCESS; 
 }
 
@@ -107,3 +123,7 @@ plugin_type Server::GetPlugin(const std::string& name)
  }
 }
 
+Statistics& Server::GetStatistics()
+{
+ return m_statistics;
+}
diff --git a/server.h b/server.h
index 11a8826..096f7f6 100644
--- a/server.h
+++ b/server.h
@@ -4,6 +4,7 @@
 
 #include "config.h"
 #include "plugin.h"
+#include "statistics.h"
 
 using namespace std::string_literals;
 
@@ -15,11 +16,12 @@ protected:
  boost::asio::io_context& m_ioc;
  const Socket& m_socket;
  plugins_container_type& m_plugins;
+ Statistics& m_statistics;
 
 public:
  static const std::string VersionString;
 
- Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& m_plugins);
+ Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics);
 
  virtual ~Server();
  virtual int start() = 0;
@@ -28,6 +30,7 @@ public:
  Config& GetConfig();
  const Socket& GetSocket();
  plugin_type GetPlugin(const std::string& name);
+ Statistics& GetStatistics();
 };
 
 int run_server(Config& config, plugins_container_type& plugins);
diff --git a/statistics.cpp b/statistics.cpp
new file mode 100644
index 0000000..19d0258
--- /dev/null
+++ b/statistics.cpp
@@ -0,0 +1,87 @@
+#include "statistics.h"
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+namespace fs = std::filesystem;
+
+namespace {
+ const fs::path statsfilepath{ "/var/lib/webserver/stats.db" };
+} // anonymous namespace
+
+Statistics::Statistics()
+{
+ std::cout << "Loading statistics..." << std::endl;
+ std::ifstream file{statsfilepath, std::ios::in | std::ios::binary};
+ if (file.is_open()) {
+  Serialization::IArchive archive{file};
+
+  archive >> mBins;
+ } else {
+  std::cerr << "Warning: Couldn't read statistics" << std::endl;
+ }
+}
+
+Statistics::~Statistics()
+{
+ std::cout << "Saving statistics..." << std::endl;
+ std::lock_guard<std::mutex> lock(mMutex);
+ std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc};
+ if (file.is_open()) {
+  Serialization::OArchive archive{file};
+
+  archive << mBins;
+ } else {
+  std::cerr << "Warning: Couldn't write statistics" << std::endl;
+ }
+}
+
+bool Statistics::Bin::expired() const
+{
+ auto now {time(nullptr)};
+
+ if (now < start_time)
+  std::runtime_error("Statistics time is in the future");
+
+ return start_time + binsize < now;
+}
+
+void Statistics::limit()
+{
+ while (mBins.size() * sizeof(Bin) > maxSize)
+  mBins.pop_front(); // discard oldest element
+}
+
+void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https)
+{
+ std::lock_guard<std::mutex> lock(mMutex);
+
+ if (mBins.empty() || mBins.back().expired()) {
+  mBins.emplace_back(Bin{static_cast<uint64_t>((time(nullptr) / binsize) * binsize), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
+ }
+
+ Bin& bin{mBins.back()};
+
+ bin.requests++;
+ if (error) bin.errors++;
+ bin.bytes_in += bytes_in;
+ bin.bytes_out += bytes_out;
+ 
+ if (ipv6) {
+  bin.requests_ipv6++;
+  if (error) bin.errors_ipv6++;
+  bin.bytes_in_ipv6 += bytes_in;
+  bin.bytes_out_ipv6 += bytes_out;
+ }
+ 
+ if (https) {
+  bin.requests_https++;
+  if (error) bin.errors_https++;
+  bin.bytes_in_https += bytes_in;
+  bin.bytes_out_https += bytes_out;
+ }
+
+ limit();
+}
+
diff --git a/statistics.h b/statistics.h
new file mode 100644
index 0000000..c4fce93
--- /dev/null
+++ b/statistics.h
@@ -0,0 +1,110 @@
+#pragma once
+
+#include "archive.h"
+
+#include <cstdint>
+#include <ctime>
+#include <deque>
+#include <iostream>
+#include <mutex>
+
+class Statistics
+{
+
+ static const int32_t binsize = 3600; // in seconds: i.e. 1 hour
+ static const size_t maxSize = 30000000; // maximum of statistics data in bytes
+
+public:
+ struct Bin
+ {
+  uint64_t start_time{};
+
+  uint64_t requests{};
+  uint64_t errors{};
+  uint64_t bytes_in{};
+  uint64_t bytes_out{};
+
+  uint64_t requests_ipv6{};
+  uint64_t errors_ipv6{};
+  uint64_t bytes_in_ipv6{};
+  uint64_t bytes_out_ipv6{};
+
+  uint64_t requests_https{};
+  uint64_t errors_https{};
+  uint64_t bytes_in_https{};
+  uint64_t bytes_out_https{};
+
+  template <class Archive>
+  void serialize (Archive& ar)
+  {
+   ar & start_time;
+
+   ar & requests;
+   ar & errors;
+   ar & bytes_in;
+   ar & bytes_out;
+
+   ar & requests_ipv6;
+   ar & errors_ipv6;
+   ar & bytes_in_ipv6;
+   ar & bytes_out_ipv6;
+
+   ar & requests_https;
+   ar & errors_https;
+   ar & bytes_in_https;
+   ar & bytes_out_https;
+  }
+
+  bool expired() const;
+
+ };
+
+private:
+ std::deque<Bin> mBins;
+ std::mutex mMutex;
+
+ void limit();
+
+public:
+ Statistics();
+ ~Statistics();
+
+ void count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https);
+};
+
+// Serialization and Deserialization as free functions
+namespace Serialization {
+
+template <class T>
+Serialization::OArchive& operator& (Serialization::OArchive& ar, std::deque<T>& deque)
+{
+ uint64_t size { deque.size() };
+
+ ar & size;
+
+ for (auto element: deque) {
+  ar & element;
+ }
+
+ return ar;
+}
+
+template <class T>
+Serialization::IArchive& operator& (Serialization::IArchive& ar, std::deque<T>& deque)
+{
+ uint64_t size {};
+
+ ar & size;
+
+ deque.clear();
+
+ for (size_t i = 0; i < size; i++) {
+  T element;
+  ar & element;
+  deque.push_back(element);
+ }
+
+ return ar;
+}
+
+}