summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRoland Reichwein <mail@reichwein.it>2020-04-26 13:01:19 +0200
committerRoland Reichwein <mail@reichwein.it>2020-04-26 13:01:19 +0200
commitaebe139d00b44684158edb3616da5c37b12db6d1 (patch)
treef37eaa7a59523f2e7db42875e65f2ca2714c0e9d
parent91073ca08f5f9582196712ae6753714f3333979f (diff)
Added stats output
-rw-r--r--Makefile2
-rw-r--r--TODO2
-rw-r--r--plugins/statistics/Makefile124
-rw-r--r--plugins/statistics/statistics.cpp97
-rw-r--r--plugins/statistics/statistics.h21
-rw-r--r--response.cpp63
-rw-r--r--server.cpp15
-rw-r--r--statistics.cpp69
-rw-r--r--statistics.h5
-rw-r--r--webserver.conf4
10 files changed, 353 insertions, 49 deletions
diff --git a/Makefile b/Makefile
index 5b84f33..41c83c4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
DISTROS=debian10
VERSION=$(shell dpkg-parsechangelog --show-field Version)
PROJECTNAME=webserver
-PLUGINS=static-files webbox cgi weblog # fcgi
+PLUGINS=static-files webbox cgi weblog statistics # fcgi
CXX=clang++-10
diff --git a/TODO b/TODO
index da9cf7c..d5dbf8b 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,4 @@
+Upload: read: body limit exceeded
weblog: blättern
weblog: link consistency check (cron?)
weblog: style: zitate
@@ -5,7 +6,6 @@ Integrate into Debian
Ubuntu version
Speed up config.GetPath
read: The socket was closed due to a timeout
-print statistics
webbox: Info if not selected: all
webbox: Copy function
crypt pws
diff --git a/plugins/statistics/Makefile b/plugins/statistics/Makefile
new file mode 100644
index 0000000..48c2e8c
--- /dev/null
+++ b/plugins/statistics/Makefile
@@ -0,0 +1,124 @@
+DISTROS=debian10
+VERSION=$(shell dpkg-parsechangelog --show-field Version)
+PROJECTNAME=statistics
+
+CXX=clang++-10
+
+ifeq ($(shell which $(CXX)),)
+CXX=clang++
+endif
+
+ifeq ($(shell which $(CXX)),)
+CXX=g++-9
+endif
+
+ifeq ($(CXXFLAGS),)
+#CXXFLAGS=-O2 -DNDEBUG
+CXXFLAGS=-O0 -g -D_DEBUG
+endif
+# -fprofile-instr-generate -fcoverage-mapping
+# gcc:--coverage
+
+CXXFLAGS+= -Wall -I.
+
+CXXFLAGS+= -pthread -fvisibility=hidden -fPIC
+ifeq ($(CXX),clang++-10)
+CXXFLAGS+=-std=c++20 #-stdlib=libc++
+else
+CXXFLAGS+=-std=c++17
+endif
+
+CXXTESTFLAGS=-Igoogletest/include -Igooglemock/include/ -Igoogletest -Igooglemock
+
+LIBS=\
+-lboost_context \
+-lboost_coroutine \
+-lboost_program_options \
+-lboost_system \
+-lboost_thread \
+-lboost_filesystem \
+-lboost_regex \
+-lpthread \
+-lssl -lcrypto \
+-ldl
+
+ifeq ($(CXX),clang++-10)
+LIBS+= \
+-fuse-ld=lld-10 \
+-lstdc++
+#-lc++ \
+#-lc++abi
+#-lc++fs
+#-lstdc++fs
+else
+LIBS+= \
+-lstdc++ \
+-lstdc++fs
+endif
+
+PROGSRC=\
+ statistics.cpp
+
+TESTSRC=\
+ test-webserver.cpp \
+ googlemock/src/gmock-all.cpp \
+ googletest/src/gtest-all.cpp \
+ $(PROGSRC)
+
+SRC=$(PROGSRC)
+
+all: $(PROJECTNAME).so
+
+# testsuite ----------------------------------------------
+test-$(PROJECTNAME): $(TESTSRC:.cpp=.o)
+ $(CXX) $(CXXFLAGS) $^ $(LIBS) -o $@
+
+$(PROJECTNAME).so: $(SRC:.cpp=.o)
+ $(CXX) -shared $(CXXFLAGS) $^ $(LIBS) -o $@
+
+dep: $(TESTSRC:.cpp=.d)
+
+%.d: %.cpp
+ $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -MM -MP -MF $@ -c $<
+
+%.o: %.cpp %.d
+ $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@
+
+googletest/src/%.o: googletest/src/%.cc
+ $(CXX) $(CXXFLAGS) $(CXXTESTFLAGS) -c $< -o $@
+
+# dependencies
+
+ADD_DEP=Makefile
+
+install:
+ mkdir -p $(DESTDIR)/usr/lib/webserver/plugins
+ cp $(PROJECTNAME).so $(DESTDIR)/usr/lib/webserver/plugins
+
+# misc ---------------------------------------------------
+deb:
+ # build binary deb package
+ dpkg-buildpackage -us -uc -rfakeroot
+
+deb-src:
+ dpkg-source -b .
+
+$(DISTROS): deb-src
+ sudo pbuilder build --basetgz /var/cache/pbuilder/$@.tgz --buildresult result/$@ ../webserver_$(VERSION).dsc ; \
+
+debs: $(DISTROS)
+
+clean:
+ -rm -f test-$(PROJECTNAME) $(PROJECTNAME)
+ -find . -name '*.o' -o -name '*.so' -o -name '*.d' -o -name '*.gcno' -o -name '*.gcda' | xargs rm -f
+
+zip: clean
+ -rm -f ../$(PROJECTNAME).zip
+ zip -r ../$(PROJECTNAME).zip *
+ ls -l ../$(PROJECTNAME).zip
+
+
+
+.PHONY: clean all zip install deb deb-src debs all $(DISTROS)
+
+-include $(wildcard $(SRC:.cpp=.d))
diff --git a/plugins/statistics/statistics.cpp b/plugins/statistics/statistics.cpp
new file mode 100644
index 0000000..03f4c94
--- /dev/null
+++ b/plugins/statistics/statistics.cpp
@@ -0,0 +1,97 @@
+#include "statistics.h"
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/coroutine2/coroutine.hpp>
+#include <boost/process.hpp>
+
+#include <algorithm>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+
+using namespace std::string_literals;
+namespace bp = boost::process;
+namespace fs = std::filesystem;
+
+namespace {
+
+ // Used to return errors by generating response page and HTTP status code
+ std::string HttpStatus(std::string status, std::string message, std::function<plugin_interface_setter_type>& SetResponseHeader)
+ {
+ SetResponseHeader("status", status);
+ SetResponseHeader("content_type", "text/html");
+ return status + " " + message;
+ }
+
+} // anonymous namespace
+
+std::string statistics_plugin::name()
+{
+ return "statistics";
+}
+
+statistics_plugin::statistics_plugin()
+{
+ //std::cout << "Plugin constructor" << std::endl;
+}
+
+statistics_plugin::~statistics_plugin()
+{
+ //std::cout << "Plugin destructor" << std::endl;
+}
+
+std::string statistics_plugin::generate_page(
+ std::function<std::string(const std::string& key)>& GetServerParam,
+ std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...)
+ std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string
+)
+{
+ try {
+ // Request path must not contain "..".
+ std::string rel_target{GetRequestParam("rel_target")};
+ size_t query_pos{rel_target.find("?")};
+ if (query_pos != rel_target.npos)
+ rel_target = rel_target.substr(0, query_pos);
+
+ std::string target{GetRequestParam("target")};
+ if (rel_target.find("..") != std::string::npos) {
+ return HttpStatus("400", "Illegal request: "s + target, SetResponseHeader);
+ }
+
+ // Build the path to the requested file
+ std::string doc_root{GetRequestParam("doc_root")};
+ fs::path path {fs::path{doc_root} / rel_target};
+ if (target.size() && target.back() != '/' && fs::is_directory(path)) {
+ std::string location{GetRequestParam("location") + "/"s};
+ SetResponseHeader("location", location);
+ return HttpStatus("301", "Correcting directory path", SetResponseHeader);
+ }
+
+ try {
+ SetResponseHeader("content_type", "text/html");
+
+ std::string header {"<!DOCTYPE html><html><head>"
+ "<meta charset=\"utf-8\"/>"
+ "<title>Webserver Statistics</title>"
+ "</head><body>"};
+ std::string footer{"<br/><br/><br/></body></html>"};
+
+ std::string result{header};
+
+ result += "<h1>Webserver Statistics</h1>";
+ result += "<pre>"s + GetServerParam("statistics") + "</pre>"s;
+
+ result += footer;
+
+ return result;
+ } catch (const std::exception& ex) {
+ return HttpStatus("500", "Statistics error: "s + ex.what(), SetResponseHeader);
+ }
+
+ } catch (const std::exception& ex) {
+ return HttpStatus("500", "Unknown Error: "s + ex.what(), SetResponseHeader);
+ }
+}
+
diff --git a/plugins/statistics/statistics.h b/plugins/statistics/statistics.h
new file mode 100644
index 0000000..5db309b
--- /dev/null
+++ b/plugins/statistics/statistics.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "../../plugin_interface.h"
+
+class statistics_plugin: public webserver_plugin_interface
+{
+public:
+ statistics_plugin();
+ ~statistics_plugin();
+
+ std::string name();
+ std::string generate_page(
+ std::function<std::string(const std::string& key)>& GetServerParam,
+ std::function<std::string(const std::string& key)>& GetRequestParam, // request including body (POST...)
+ std::function<void(const std::string& key, const std::string& value)>& SetResponseHeader // to be added to result string
+ );
+
+};
+
+extern "C" BOOST_SYMBOL_EXPORT statistics_plugin webserver_plugin;
+statistics_plugin webserver_plugin;
diff --git a/response.cpp b/response.cpp
index c5ba426..696b859 100644
--- a/response.cpp
+++ b/response.cpp
@@ -79,10 +79,11 @@ bool is_ipv6_address(const std::string& addr)
std::unordered_map<std::string, std::function<std::string(Server&)>> GetServerParamFunctions{
// following are the supported fields:
- {"version", [](Server& server) { return Server::VersionString; }},
{"address", [](Server& server) { return server.GetSocket().address; }},
{"ipv6", [](Server& server) { return is_ipv6_address(server.GetSocket().address) ? "yes" : "no"; }},
{"port", [](Server& server) { return server.GetSocket().port; }},
+ {"statistics", [](Server& server) { return server.GetStatistics().getValues(); }},
+ {"version", [](Server& server) { return Server::VersionString; }},
};
std::string GetServerParam(const std::string& key, Server& server)
@@ -233,42 +234,34 @@ mime_type(beast::string_view path)
return "application/text";
}
-// Used to return errors by generating response page and HTTP status code
response_type HttpStatus(std::string status, std::string message, response_type& res)
{
- res.result(unsigned(stoul(status)));
- res.set(http::field::content_type, "text/html");
- if (res.result_int() == 401)
- res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\"");
- res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s;
- res.prepare_payload();
+ if (status != "200") { // already handled at res init
+ res.result(unsigned(stoul(status)));
+ res.set(http::field::content_type, "text/html");
+ if (res.result_int() == 401)
+ res.set(http::field::www_authenticate, "Basic realm=\"Webbox Login\"");
+ res.body() = "<html><body><h1>"s + Server::VersionString + " Error</h1><p>"s + status + " "s + message + "</p></body></html>"s;
+ res.prepare_payload();
+ }
return res;
}
-// Do statistics at end of response generation, handle all exit paths via RAII
-class StatisticsGuard
+// Used to return errors by generating response page and HTTP status code
+response_type HttpStatusAndStats(std::string status, std::string message, RequestContext& req_ctx, response_type& res)
{
- request_type& mReq;
- response_type& mRes;
- Server& mServer;
-public:
- StatisticsGuard(request_type& req, response_type& res, Server& server)
- : mReq(req)
- , mRes(res)
- , mServer(server)
- {
- }
+ HttpStatus(status, message, res);
- ~StatisticsGuard()
- {
- mServer.GetStatistics().count(mReq.body().size(),
- mRes.body().size(),
- mRes.result_int() == 200,
- is_ipv6_address(mServer.GetSocket().address),
- mServer.GetSocket().protocol == SocketProtocol::HTTPS);
- }
-};
+ req_ctx.GetServer().GetStatistics().count(
+ req_ctx.GetReq().body().size(),
+ res.body().size(),
+ res.result_int() != 200,
+ is_ipv6_address(req_ctx.GetServer().GetSocket().address),
+ req_ctx.GetServer().GetSocket().protocol == SocketProtocol::HTTPS);
+
+ return std::move(res);
+}
} // anonymous namespace
@@ -279,8 +272,6 @@ response_type generate_response(request_type& req, Server& server)
res.set(http::field::content_type, mime_type(extend_index_html(std::string(req.target()))));
res.keep_alive(req.keep_alive());
- StatisticsGuard statsGuard{req, res, server};
-
try {
RequestContext req_ctx{req, server}; // can throw std::out_of_range
@@ -288,21 +279,21 @@ response_type generate_response(request_type& req, Server& server)
if (auth.size() != 0) {
std::string authorization{req[http::field::authorization]};
if (authorization.substr(0, 6) != "Basic "s)
- return HttpStatus("401", "Bad Authorization Type", res);
+ return HttpStatusAndStats("401", "Bad Authorization Type", req_ctx, res);
authorization = authorization.substr(6);
authorization = decode64(authorization);
size_t pos {authorization.find(':')};
if (pos == authorization.npos)
- return HttpStatus("401", "Bad Authorization Encoding", res);
+ return HttpStatusAndStats("401", "Bad Authorization Encoding", req_ctx, res);
std::string login{authorization.substr(0, pos)};
std::string password{authorization.substr(pos + 1)};
auto it {auth.find(login)};
if (it == auth.end() || it->second != password)
- return HttpStatus("401", "Bad Authorization", res);
+ return HttpStatusAndStats("401", "Bad Authorization", req_ctx, res);
}
plugin_type plugin{req_ctx.GetPlugin()};
@@ -318,8 +309,8 @@ response_type generate_response(request_type& req, Server& server)
res.body() = res_data;
res.prepare_payload();
}
-
- return res;
+
+ return HttpStatusAndStats("200", "OK", req_ctx, res);
} catch(const std::out_of_range& ex) {
return HttpStatus("400", "Bad request: Host "s + std::string{req["host"]} + ":"s + std::string{req.target()} + " unknown"s, res);
} catch(const std::exception& ex) {
diff --git a/server.cpp b/server.cpp
index 71f39ac..5d1609d 100644
--- a/server.cpp
+++ b/server.cpp
@@ -20,6 +20,7 @@
#include <boost/config.hpp>
#include <exception>
+#include <functional>
#include <iostream>
#include <thread>
#include <vector>
@@ -39,6 +40,10 @@ using tcp = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>
const std::string Server::VersionString{ "Reichwein.IT Webserver "s + std::string{VERSION} };
+namespace {
+ const int32_t stats_timer_seconds { 24 * 60 * 60 }; // save stats once a day
+} // anonymous namespace
+
Server::Server(Config& config, boost::asio::io_context& ioc, const Socket& socket, plugins_container_type& plugins, Statistics& statistics)
: m_config(config)
, m_ioc(ioc)
@@ -66,6 +71,16 @@ int run_server(Config& config, plugins_container_type& plugins)
ioc.stop();
});
+ // Save stats once a day
+ boost::asio::steady_timer stats_save_timer(ioc, boost::asio::chrono::seconds(stats_timer_seconds));
+ std::function<void(const boost::system::error_code&)> stats_callback =
+ [&](const boost::system::error_code& error){
+ stats.save();
+ stats_save_timer.expires_at(stats_save_timer.expires_at() + boost::asio::chrono::seconds(stats_timer_seconds));
+ stats_save_timer.async_wait(stats_callback);
+ };
+ stats_save_timer.async_wait(stats_callback);
+
std::vector<std::shared_ptr<Server>> servers;
const auto& sockets {config.Sockets()};
diff --git a/statistics.cpp b/statistics.cpp
index 19d0258..3fb99a3 100644
--- a/statistics.cpp
+++ b/statistics.cpp
@@ -5,13 +5,15 @@
#include <iostream>
namespace fs = std::filesystem;
+using namespace std::string_literals;
namespace {
const fs::path statsfilepath{ "/var/lib/webserver/stats.db" };
} // anonymous namespace
-Statistics::Statistics()
+void Statistics::load()
{
+ std::lock_guard<std::mutex> lock(mMutex);
std::cout << "Loading statistics..." << std::endl;
std::ifstream file{statsfilepath, std::ios::in | std::ios::binary};
if (file.is_open()) {
@@ -21,22 +23,38 @@ Statistics::Statistics()
} else {
std::cerr << "Warning: Couldn't read statistics" << std::endl;
}
+
+ mChanged = false;
}
-Statistics::~Statistics()
+void Statistics::save()
{
- std::cout << "Saving statistics..." << std::endl;
- std::lock_guard<std::mutex> lock(mMutex);
- std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc};
- if (file.is_open()) {
- Serialization::OArchive archive{file};
-
- archive << mBins;
- } else {
- std::cerr << "Warning: Couldn't write statistics" << std::endl;
+ if (mChanged) {
+ std::lock_guard<std::mutex> lock(mMutex);
+ std::cout << "Saving statistics..." << std::endl;
+ std::ofstream file{statsfilepath, std::ios::out | std::ios::binary | std::ios::trunc};
+ if (file.is_open()) {
+ Serialization::OArchive archive{file};
+
+ archive << mBins;
+ } else {
+ std::cerr << "Warning: Couldn't write statistics" << std::endl;
+ }
+
+ mChanged = false;
}
}
+Statistics::Statistics()
+{
+ load();
+}
+
+Statistics::~Statistics()
+{
+ save();
+}
+
bool Statistics::Bin::expired() const
{
auto now {time(nullptr)};
@@ -57,6 +75,8 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6,
{
std::lock_guard<std::mutex> lock(mMutex);
+ mChanged = true;
+
if (mBins.empty() || mBins.back().expired()) {
mBins.emplace_back(Bin{static_cast<uint64_t>((time(nullptr) / binsize) * binsize), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
}
@@ -85,3 +105,30 @@ void Statistics::count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6,
limit();
}
+std::string Statistics::getValues()
+{
+ std::lock_guard<std::mutex> lock(mMutex);
+
+ std::string result;
+
+ for (const auto& bin: mBins) {
+ result += std::to_string(bin.start_time) + ","s +
+
+ std::to_string(bin.requests) + ","s +
+ std::to_string(bin.errors) + ","s +
+ std::to_string(bin.bytes_in) + ","s +
+ std::to_string(bin.bytes_out) + ","s +
+
+ std::to_string(bin.requests_ipv6) + ","s +
+ std::to_string(bin.errors_ipv6) + ","s +
+ std::to_string(bin.bytes_in_ipv6) + ","s +
+ std::to_string(bin.bytes_out_ipv6) + ","s +
+
+ std::to_string(bin.requests_https) + ","s +
+ std::to_string(bin.errors_https) + ","s +
+ std::to_string(bin.bytes_in_https) + ","s +
+ std::to_string(bin.bytes_out_https) + "\n"s;
+ }
+
+ return result;
+}
diff --git a/statistics.h b/statistics.h
index c4fce93..7e4da7e 100644
--- a/statistics.h
+++ b/statistics.h
@@ -60,9 +60,11 @@ public:
};
private:
+ bool mChanged{};
std::deque<Bin> mBins;
std::mutex mMutex;
+ void load();
void limit();
public:
@@ -70,6 +72,9 @@ public:
~Statistics();
void count(size_t bytes_in, size_t bytes_out, bool error, bool ipv6, bool https);
+ void save();
+
+ std::string getValues();
};
// Serialization and Deserialization as free functions
diff --git a/webserver.conf b/webserver.conf
index 52075a2..5adc9ba 100644
--- a/webserver.conf
+++ b/webserver.conf
@@ -38,6 +38,10 @@
<WEBLOG_DESCRIPTION>Roland Reichweins Blog</WEBLOG_DESCRIPTION>
<WEBLOG_KEYWORDS>Roland Reichwein, Blog</WEBLOG_KEYWORDS>
</path>
+ <path requested="/stats">
+ <plugin>statistics</plugin>
+ <target></target>
+ </path>
<path requested="/cgi-bin">
<plugin>cgi</plugin>
<target>/home/ernie/code/webserver/cgi-bin</target>