diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | TODO | 1 | ||||
| -rw-r--r-- | archive.h | 206 | ||||
| -rw-r--r-- | debian/changelog | 1 | ||||
| -rw-r--r-- | debian/webserver.dirs | 1 | ||||
| -rw-r--r-- | http.cpp | 4 | ||||
| -rw-r--r-- | http.h | 2 | ||||
| -rw-r--r-- | https.cpp | 4 | ||||
| -rw-r--r-- | https.h | 2 | ||||
| -rw-r--r-- | response.cpp | 31 | ||||
| -rw-r--r-- | server.cpp | 38 | ||||
| -rw-r--r-- | server.h | 5 | ||||
| -rw-r--r-- | statistics.cpp | 87 | ||||
| -rw-r--r-- | statistics.h | 110 |
14 files changed, 473 insertions, 20 deletions
@@ -67,6 +67,7 @@ PROGSRC=\ plugin.cpp \ privileges.cpp \ response.cpp \ + statistics.cpp \ server.cpp TESTSRC=\ @@ -2,6 +2,7 @@ weblog: blättern weblog: link consistency check (cron?) weblog: style: zitate Integrate into Debian +debian: restart server on install/update Ubuntu version Speed up config.GetPath read: The socket was closed due to a timeout diff --git a/archive.h b/archive.h new file mode 100644 index 0000000..a98921a --- /dev/null +++ b/archive.h @@ -0,0 +1,206 @@ +#pragma once + +#include <boost/coroutine2/coroutine.hpp> +#include <boost/endian/conversion.hpp> + +#include <cstdint> +#include <ostream> +#include <istream> +#include <sstream> +#include <string> +#include <vector> + +typedef boost::coroutines2::coroutine<void> coro_t; + +// Serialization, similar to Boost Serialization +// but for portable binary archive +// using big endian coding (network byte order) +namespace Serialization { + +class OArchive +{ +public: + OArchive(std::ostream& os): os(os) {} + ~OArchive() {} + + template<class T> + OArchive& operator &(T& v) { + v.serialize(*this); + + return *this; + }; + + template <class T> + OArchive& write_fundamental(T& v) + { + T value = boost::endian::native_to_big(v); + os.write((char*)&value, sizeof(value)); + return *this; + } + + OArchive& operator &(uint8_t& v) + { + return write_fundamental(v); + }; + + OArchive& operator &(uint16_t& v) + { + return write_fundamental(v); + }; + + OArchive& operator &(uint32_t& v) + { + return write_fundamental(v); + }; + + OArchive& operator &(uint64_t& v) + { + return write_fundamental(v); + }; + + OArchive& operator &(int64_t& v) + { + return write_fundamental(*reinterpret_cast<uint64_t*>(v)); + }; + + OArchive& operator &(std::vector<uint8_t>& v) + { + uint32_t size = static_cast<uint32_t>(v.size()); + *this & size; + os.write((char*)v.data(), size); + return *this; + }; + + OArchive& operator &(std::string& v) + { + uint32_t size = static_cast<uint32_t>(v.size()); + *this & size; + os.write((char*)v.data(), v.size()); + return *this; + }; + +private: + std::ostream &os; +}; + +class IArchive +{ +public: + IArchive(std::istream& is): is(is) {} + IArchive(std::stringstream& is, coro_t::pull_type& coro) : is(is), mStringStream(&is), mCoro(&coro) {} + ~IArchive() {} + + template<class T> + IArchive& operator &(T& v) + { + v.serialize(*this); + + return *this; + }; + + template <class T> + IArchive& read_fundamental(T& v) + { + // in coroutine case, wait for input, if necessary + if (mCoro && mStringStream) { + while (mStringStream->tellp() - mStringStream->tellg() < sizeof(v)) { + (*mCoro)(); + } + } + + // now, we have enough bytes available + T value; + is.read((char*)&value, sizeof(value)); + v = boost::endian::big_to_native(value); + return *this; + } + + IArchive& operator &(uint8_t& v) + { + return read_fundamental(v); + }; + + IArchive& operator &(uint16_t& v) + { + return read_fundamental(v); + }; + + IArchive& operator &(uint32_t& v) + { + return read_fundamental(v); + }; + + IArchive& operator &(uint64_t& v) + { + return read_fundamental(v); + }; + + IArchive& operator &(int64_t& v) + { + uint64_t uv; + read_fundamental(uv); + v = *reinterpret_cast<int64_t*>(uv); + return *this; + }; + + template <class T> + IArchive& read_bytes_vector(T& v) + { + uint32_t size; + *this & size; + + v.resize(size); + + // in coroutine case, wait for input, if necessary + if (mCoro && mStringStream) { + while (mStringStream->tellp() - mStringStream->tellg() < size) { + (*mCoro)(); + } + } + + // now, we have enough bytes available + is.read((char*)v.data(), size); + return *this; + } + + IArchive& operator &(std::vector<uint8_t>& v) + { + return read_bytes_vector(v); + }; + + IArchive& operator &(std::string& v) + { + return read_bytes_vector(v); + }; + +private: + std::istream &is; + std::stringstream* mStringStream{ }; // for i/o sizes access + coro_t::pull_type* mCoro{ }; // optional for coroutine +}; + +// - Free functions ---------------------------------------------------------- + +template<class Archive, class T> +void serialize(Archive& ar, T& v) +{ + ar & v; +} + +template<class T> +OArchive& operator <<(OArchive& ar, T& v) +{ + serialize(ar, v); + + return ar; +}; + +template<class T> +IArchive& operator >>(IArchive& ar, T& v) +{ + serialize(ar, v); + + return ar; +}; + +} diff --git a/debian/changelog b/debian/changelog index dc8df04..5b7f88a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ webserver (1.3) unstable; urgency=medium * Updated weblog + * Added statistics -- Roland Reichwein <rr@antcom.de> Sat, 25 Apr 2020 12:35:44 +0200 diff --git a/debian/webserver.dirs b/debian/webserver.dirs new file mode 100644 index 0000000..23315d6 --- /dev/null +++ b/debian/webserver.dirs @@ -0,0 +1 @@ +var/lib/webserver @@ -357,8 +357,8 @@ private: namespace HTTP { - Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins) - : ::Server(config, ioc, socket, plugins) + Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics) + : ::Server(config, ioc, socket, plugins, statistics) { } @@ -11,7 +11,7 @@ namespace HTTP { class Server: public ::Server { public: - Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins); + Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics); virtual ~Server(); int start() override; }; @@ -630,8 +630,8 @@ int servername_callback(SSL *s, int *al, void *arg) namespace HTTPS { -Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins) - : ::Server(config, ioc, socket, plugins) +Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics) + : ::Server(config, ioc, socket, plugins, statistics) { // initial dummy, before we can add specific ctx w/ certificate std::shared_ptr<ssl::context> ctx_dummy{std::make_shared<ssl::context>(tls_method)}; @@ -40,7 +40,7 @@ private: ctx_type m_ctx; public: - Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins); + Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics); virtual ~Server(); int start() override; diff --git a/response.cpp b/response.cpp index e1b6c05..c5ba426 100644 --- a/response.cpp +++ b/response.cpp @@ -246,6 +246,30 @@ response_type HttpStatus(std::string status, std::string message, response_type& return res; } +// Do statistics at end of response generation, handle all exit paths via RAII +class StatisticsGuard +{ + request_type& mReq; + response_type& mRes; + Server& mServer; +public: + StatisticsGuard(request_type& req, response_type& res, Server& server) + : mReq(req) + , mRes(res) + , mServer(server) + { + } + + ~StatisticsGuard() + { + mServer.GetStatistics().count(mReq.body().size(), + mRes.body().size(), + mRes.result_int() == 200, + is_ipv6_address(mServer.GetSocket().address), + mServer.GetSocket().protocol == SocketProtocol::HTTPS); + } +}; + } // anonymous namespace response_type generate_response(request_type& req, Server& server) @@ -255,6 +279,8 @@ response_type generate_response(request_type& req, Server& server) res.set(http::field::content_type, mime_type(extend_index_html(std::string(req.target())))); res.keep_alive(req.keep_alive()); + StatisticsGuard statsGuard{req, res, server}; + try { RequestContext req_ctx{req, server}; // can throw std::out_of_range @@ -275,11 +301,8 @@ response_type generate_response(request_type& req, Server& server) std::string password{authorization.substr(pos + 1)}; auto it {auth.find(login)}; - if (it == auth.end()) + if (it == auth.end() || it->second != password) return HttpStatus("401", "Bad Authorization", res); - - if (it->second != password) - return HttpStatus("401", "Bad Authorization", res); // should be same message as previous one to prevent login guessing } plugin_type plugin{req_ctx.GetPlugin()}; @@ -15,6 +15,7 @@ #include <boost/asio/ssl/stream.hpp> #endif #include <boost/asio/dispatch.hpp> +#include <boost/asio/signal_set.hpp> #include <boost/asio/strand.hpp> #include <boost/config.hpp> @@ -28,6 +29,7 @@ #include "http.h" #include "https.h" #include "privileges.h" +#include "statistics.h" namespace beast = boost::beast; // from <boost/beast.hpp> namespace http = beast::http; // from <boost/beast/http.hpp> @@ -37,11 +39,12 @@ using tcp = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp> const std::string Server::VersionString{ "Reichwein.IT Webserver "s + std::string{VERSION} }; -Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins) +Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics) : m_config(config) , m_ioc(ioc) , m_socket(socket) , m_plugins(plugins) + , m_statistics(statistics) { } @@ -51,18 +54,26 @@ Server::~Server() int run_server(Config& config, plugins_container_type& plugins) { + Statistics stats; + auto const threads = std::max<int>(1, config.Threads()); boost::asio::io_context ioc{threads}; + boost::asio::signal_set signals(ioc, SIGINT, SIGTERM); + signals.async_wait([&](const boost::system::error_code& error, int signal_number){ + std::cout << "Terminating via signal " << signal_number << std::endl; + ioc.stop(); + }); + std::vector<std::shared_ptr<Server>> servers; const auto& sockets {config.Sockets()}; for (const auto& socket: sockets) { if (socket.protocol == SocketProtocol::HTTP) { - servers.push_back(std::make_shared<HTTP::Server>(config, ioc, socket, plugins)); + servers.push_back(std::make_shared<HTTP::Server>(config, ioc, socket, plugins, stats)); } else { - servers.push_back(std::make_shared<HTTPS::Server>(config, ioc, socket, plugins)); + servers.push_back(std::make_shared<HTTPS::Server>(config, ioc, socket, plugins, stats)); } servers.back()->start(); } @@ -73,14 +84,19 @@ int run_server(Config& config, plugins_container_type& plugins) // Run the I/O service on the requested number of threads std::vector<std::thread> v; v.reserve(threads - 1); - for(auto i = threads - 1; i > 0; --i) - v.emplace_back( - [&ioc] - { - ioc.run(); - }); + for (auto i = threads - 1; i > 0; --i) { + v.emplace_back( + [&ioc] + { + ioc.run(); + }); + } ioc.run(); + for (auto& t: v) { + t.join(); + } + return EXIT_SUCCESS; } @@ -107,3 +123,7 @@ plugin_type Server::GetPlugin(const std::string& name) } } +Statistics& Server::GetStatistics() +{ + return m_statistics; +} @@ -4,6 +4,7 @@ #include "config.h" #include "plugin.h" +#include "statistics.h" using namespace std::string_literals; @@ -15,11 +16,12 @@ protected: boost::asio::io_context& m_ioc; const Socket& m_socket; plugins_container_type& m_plugins; + Statistics& m_statistics; public: static const std::string VersionString; - Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& m_plugins); + Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics); virtual ~Server(); virtual int start() = 0; @@ -28,6 +30,7 @@ public: Config& GetConfig(); const Socket& GetSocket(); plugin_type GetPlugin(const std::string& name); + Statistics& GetStatistics(); }; int run_server(Config& config, plugins_container_type& plugins); diff --git a/statistics.cpp b/statistics.cpp new file mode 100644 index 0000000..19d0258 --- /dev/null +++ b/statistics.cpp @@ -0,0 +1,87 @@ +#include "statistics.h" + +#include <filesystem> +#include <fstream> +#include <iostream> + +namespace fs = std::filesystem; + +namespace { + const fs::path statsfilepath{ "/var/lib/webserver/stats.db" }; +} // anonymous namespace + +Statistics::Statistics() +{ + std::cout << "Loading statistics..." << std::endl; + std::ifstream file{statsfilepath, std::ios::in | std::ios::binary}; + if (file.is_open()) { + Serialization::IArchive archive{file}; + + archive >> mBins; + } else { + std::cerr << "Warning: Couldn't read statistics" << std::endl; + } +} + +Statistics::~Statistics() +{ + std::cout << "Saving statistics..." << std::endl; + std::lock_guard<std::mutex> lock(mMutex); + std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc}; + if (file.is_open()) { + Serialization::OArchive archive{file}; + + archive << mBins; + } else { + std::cerr << "Warning: Couldn't write statistics" << std::endl; + } +} + +bool Statistics::Bin::expired() const +{ + auto now {time(nullptr)}; + + if (now < start_time) + std::runtime_error("Statistics time is in the future"); + + return start_time + binsize < now; +} + +void Statistics::limit() +{ + while (mBins.size() * sizeof(Bin) > maxSize) + mBins.pop_front(); // discard oldest element +} + +void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https) +{ + std::lock_guard<std::mutex> lock(mMutex); + + if (mBins.empty() || mBins.back().expired()) { + mBins.emplace_back(Bin{static_cast<uint64_t>((time(nullptr) / binsize) * binsize), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + } + + Bin& bin{mBins.back()}; + + bin.requests++; + if (error) bin.errors++; + bin.bytes_in += bytes_in; + bin.bytes_out += bytes_out; + + if (ipv6) { + bin.requests_ipv6++; + if (error) bin.errors_ipv6++; + bin.bytes_in_ipv6 += bytes_in; + bin.bytes_out_ipv6 += bytes_out; + } + + if (https) { + bin.requests_https++; + if (error) bin.errors_https++; + bin.bytes_in_https += bytes_in; + bin.bytes_out_https += bytes_out; + } + + limit(); +} + diff --git a/statistics.h b/statistics.h new file mode 100644 index 0000000..c4fce93 --- /dev/null +++ b/statistics.h @@ -0,0 +1,110 @@ +#pragma once + +#include "archive.h" + +#include <cstdint> +#include <ctime> +#include <deque> +#include <iostream> +#include <mutex> + +class Statistics +{ + + static const int32_t binsize = 3600; // in seconds: i.e. 1 hour + static const size_t maxSize = 30000000; // maximum of statistics data in bytes + +public: + struct Bin + { + uint64_t start_time{}; + + uint64_t requests{}; + uint64_t errors{}; + uint64_t bytes_in{}; + uint64_t bytes_out{}; + + uint64_t requests_ipv6{}; + uint64_t errors_ipv6{}; + uint64_t bytes_in_ipv6{}; + uint64_t bytes_out_ipv6{}; + + uint64_t requests_https{}; + uint64_t errors_https{}; + uint64_t bytes_in_https{}; + uint64_t bytes_out_https{}; + + template <class Archive> + void serialize (Archive& ar) + { + ar & start_time; + + ar & requests; + ar & errors; + ar & bytes_in; + ar & bytes_out; + + ar & requests_ipv6; + ar & errors_ipv6; + ar & bytes_in_ipv6; + ar & bytes_out_ipv6; + + ar & requests_https; + ar & errors_https; + ar & bytes_in_https; + ar & bytes_out_https; + } + + bool expired() const; + + }; + +private: + std::deque<Bin> mBins; + std::mutex mMutex; + + void limit(); + +public: + Statistics(); + ~Statistics(); + + void count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https); +}; + +// Serialization and Deserialization as free functions +namespace Serialization { + +template <class T> +Serialization::OArchive& operator& (Serialization::OArchive& ar, std::deque<T>& deque) +{ + uint64_t size { deque.size() }; + + ar & size; + + for (auto element: deque) { + ar & element; + } + + return ar; +} + +template <class T> +Serialization::IArchive& operator& (Serialization::IArchive& ar, std::deque<T>& deque) +{ + uint64_t size {}; + + ar & size; + + deque.clear(); + + for (size_t i = 0; i < size; i++) { + T element; + ar & element; + deque.push_back(element); + } + + return ar; +} + +} |