From 4aeab7931182cb1c35bd5c52b58d71b30c32674d Mon Sep 17 00:00:00 2001 From: Roland Reichwein Date: Sat, 5 Nov 2022 13:49:53 +0100 Subject: Initial files, WIP --- Makefile | 90 +++++++++++++++++++ debian/README.Debian | 44 ++++++++++ debian/changelog | 5 ++ debian/compat | 1 + debian/control | 16 ++++ debian/copyright | 4 + debian/rules | 4 + debian/source/format | 1 + debian/whiteboard.service | 13 +++ file.cpp | 46 ++++++++++ file.h | 15 ++++ html/index.html | 60 +++++++++++++ html/whiteboard.css | 69 +++++++++++++++ html/whiteboard.js | 116 +++++++++++++++++++++++++ start.sh | 5 ++ webserver.conf.example | 8 ++ whiteboard.cpp | 215 ++++++++++++++++++++++++++++++++++++++++++++++ 17 files changed, 712 insertions(+) create mode 100755 Makefile create mode 100644 debian/README.Debian create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/whiteboard.service create mode 100644 file.cpp create mode 100644 file.h create mode 100644 html/index.html create mode 100644 html/whiteboard.css create mode 100644 html/whiteboard.js create mode 100755 start.sh create mode 100644 webserver.conf.example create mode 100644 whiteboard.cpp diff --git a/Makefile b/Makefile new file mode 100755 index 0000000..034b024 --- /dev/null +++ b/Makefile @@ -0,0 +1,90 @@ +# +# Makefile +# +# Environment: Debian +# + +DISTROS=debian10 debian11 ubuntu2004 ubuntu2204 +VERSION=$(shell dpkg-parsechangelog --show-field Version) + +CXX=clang++-10 + +ifeq ($(shell which $(CXX)),) +CXX=clang++ +endif + +ifeq ($(shell which $(CXX)),) +CXX=g++-9 +endif + +ifeq ($(shell which $(CXX)),) +CXX=g++ +endif + +LIBS=-lfcgi -lboost_filesystem +INCLUDES=-I. +CXXFLAGS=-Wall -g -O2 -fPIC -std=c++17 -Wpedantic +HEADERS=file.h +SOURCES=$(HEADERS:.h=.cpp) +OBJECTS=$(HEADERS:.h=.o) +TARGETS=whiteboard.fcgi + +ifeq ($(CXX),clang++-10) +LIBS+= \ +-fuse-ld=lld-10 \ +-lstdc++ +#-lc++ \ +#-lc++abi +#-lc++fs +#-lstdc++fs +else +LIBS+= \ +-lstdc++ \ +-lstdc++fs +endif + +build: $(TARGETS) + +all: build + ./start.sh + +install: + mkdir -p $(DESTDIR)/usr/lib/whiteboard + cp whiteboard.fcgi $(DESTDIR)/usr/lib/whiteboard/ + + mkdir -p $(DESTDIR)/usr/lib/whiteboard/html + cp -r html/* $(DESTDIR)/usr/lib/whiteboard/html/ + + uglifyjs html/whiteboard.js -m -c > $(DESTDIR)/usr/lib/whiteboard/html/whiteboard.js + htmlmin html/index.html $(DESTDIR)/usr/lib/whiteboard/html/index.html + cleancss -o $(DESTDIR)/usr/lib/whiteboard/html/whiteboard.css html/whiteboard.css + + +whiteboard.fcgi: $(OBJECTS) + +# link +%.fcgi: %.o + $(CXX) $(LDFLAGS) $^ $(LDLIBS) $(LIBS) -o $@ + +# .cpp -> .o +%.o: %.cpp + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ + +clean: + -rm -f *.o *.fcgi + +deb: + dpkg-buildpackage + +deb-src: clean + dh_clean + dh_auto_clean + dpkg-source -b -I.git -Iresult . + +$(DISTROS): deb-src + sudo pbuilder build --basetgz /var/cache/pbuilder/$@.tgz --buildresult result/$@ ../whiteboard_$(VERSION).dsc + debsign result/$@/whiteboard_$(VERSION)_amd64.changes + +debs: $(DISTROS) + +.PHONY: clean diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..f3f2434 --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,44 @@ +whiteboard for Debian +===================== + +This package is the Debian version of whiteboard. + + +Configuration +------------- + +* You can add this to /etc/webserver.conf + + + static-files + /usr/lib/whiteboard/html + + + fcgi + 127.0.0.1:9014 + + +* Enable: + + # systemctl enable whiteboard.service + +* Start: + + # systemctl start whiteboard + +* Stop: + + # systemctl stop whiteboard + +* Query Status: + + # systemctl status whiteboard + + and observe /var/log/syslog + + +Contact +------- + +Reichwein IT + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..3db9796 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +whiteboard (1.0) unstable; urgency=medium + + * Initial release + + -- Roland Reichwein Sat, 05 Nov 2022 13:34:57 +0100 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +12 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..dc159fa --- /dev/null +++ b/debian/control @@ -0,0 +1,16 @@ +Source: whiteboard +Section: web +Priority: optional +Maintainer: Roland Reichwein +Build-Depends: debhelper (>= 12), libboost-all-dev | libboost1.71-all-dev, clang | g++-9, node-uglify, python3-pkg-resources, htmlmin, cleancss, libfcgi-dev +Standards-Version: 4.5.0 +Homepage: http://www.reichwein.it/whiteboard/ + +Package: whiteboard +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, spawn-fcgi +Recommends: webserver +Homepage: http://www.reichwein.it/whiteboard/ +Description: Web application for an collaborative editor + Whiteboard is a text editor running on an HTML5 webpage (including a server + part) that enables collaborative editing and presenting. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..5007f59 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,4 @@ +Author: Roland Reichwein , 2022 + +Both upstream source code and Debian packaging is available +under the conditions of CC0 1.0 Universal diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..2d33f6a --- /dev/null +++ b/debian/rules @@ -0,0 +1,4 @@ +#!/usr/bin/make -f + +%: + dh $@ diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/debian/whiteboard.service b/debian/whiteboard.service new file mode 100644 index 0000000..c60f3f0 --- /dev/null +++ b/debian/whiteboard.service @@ -0,0 +1,13 @@ +[Unit] +Description=Whiteboard +After=network.target + +[Service] +Type=simple +# Restart=always +ExecStart=spawn-fcgi -a 127.0.0.1 -p 9014 -n -- /usr/lib/whiteboard/whiteboard.fcgi + +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/file.cpp b/file.cpp new file mode 100644 index 0000000..47ab8be --- /dev/null +++ b/file.cpp @@ -0,0 +1,46 @@ +#include "file.h" + +#include + +namespace fs = std::filesystem; + +using namespace std::string_literals; + +std::string File::getFile(const fs::path& filename) +{ + std::ifstream file(filename.string(), std::ios::in | std::ios::binary | std::ios::ate); + + if (file.is_open()) { + std::ifstream::pos_type fileSize = file.tellg(); + file.seekg(0, std::ios::beg); + + std::string bytes(fileSize, ' '); + file.read(reinterpret_cast(bytes.data()), fileSize); + + return bytes; + + } else { + throw std::runtime_error("Opening "s + filename.string() + " for reading"); + } +} + +void File::setFile(const fs::path& filename, const std::string& s) +{ + File::setFile(filename, s.data(), s.size()); +} + +void File::setFile(const fs::path& filename, const char* data, size_t size) +{ + std::ofstream file(filename.string(), std::ios::out | std::ios::binary); + if (file.is_open()) { + file.write(data, size); + } else { + throw std::runtime_error("Opening "s + filename.string() + " for writing"); + } +} + +void File::setFile(const fs::path& filename, const std::vector& data) +{ + File::setFile(filename, reinterpret_cast(data.data()), data.size()); +} + diff --git a/file.h b/file.h new file mode 100644 index 0000000..e7e4cf6 --- /dev/null +++ b/file.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include +#include +#include + +namespace File { + +std::string getFile(const std::filesystem::path& filename); +void setFile(const std::filesystem::path& filename, const std::string& s); +void setFile(const std::filesystem::path& filename, const char* data, size_t size); +void setFile(const std::filesystem::path& filename, const std::vector& data); + +} diff --git a/html/index.html b/html/index.html new file mode 100644 index 0000000..f97b295 --- /dev/null +++ b/html/index.html @@ -0,0 +1,60 @@ + + + + + + + DownTube + + + + + +
+

+ +

+ Download internet videos as MP3 (audio) or MP4 (video). +

+ +

+ Video URL:
+

+

+ +

+ Transform to format:
+ +
+ +
+

+ +
+
 
+

+ +

+ +
+
+
+
+
+
+

+ Note: Audio download is currently limited to 30MB, Video download is limited to 300MB. +

+
+
+

Contact

+ Roland Reichwein
+ Hauptstr. 101a
+ 82008 Unterhaching
+ mail@reichwein.it
+ https://www.reichwein.it
+
+ + + + diff --git a/html/whiteboard.css b/html/whiteboard.css new file mode 100644 index 0000000..2f68794 --- /dev/null +++ b/html/whiteboard.css @@ -0,0 +1,69 @@ +body { + font-family: "sans-serif"; +} + +figcaption { + text-align: center; + font-size: 8px; + color: #808080; +} + +figure { + display: inline-block; +} + +p { + margin: 30px 0px 30px 0px; +} + +div.status { + color: #FF0000; +} + +.mobile { + width: 300px; + border-width: 80px 15px 80px 15px; + border-style: solid; + border-radius: 30px; + border-color: #000000; +} + +.logo { + display: block; + margin: 0 auto; +} + +.screenshot { + width: 400px; + border: 2px solid; + border-color: #8888AA; +} + +img.banner { + vertical-align: -5px; +} + +.button { + color:#FFFFFF; + background-color:#50B050; + text-decoration: none; + padding: 15px 20px; + font-size: 16px; + border: none; + border-radius: 6px; + cursor: pointer; +} + +@media only screen and (min-width: 1px) and (max-width: 630px) { +} + +@media only screen and (min-width: 631px) and (max-width: 950px) { +} + +@media only screen and (min-width: 951px) { + div.page { + max-width: 950px; + width: 100%; + margin: 0 auto; + } +} diff --git a/html/whiteboard.js b/html/whiteboard.js new file mode 100644 index 0000000..f79443e --- /dev/null +++ b/html/whiteboard.js @@ -0,0 +1,116 @@ +// started on main page load +function init() { + + // Connect "Enter" in text field with Button click + var url = document.getElementById("url"); + url.addEventListener("keyup", function(event) { + if (event.keyCode === 13) { + event.preventDefault(); + on_start(); + } + }); +} + +function set_status(message) { + if (message == "") + message = " "; + + document.getElementById("status").innerHTML = message; +} + +// started on button click: get filename +function on_start() { + var xhr = new XMLHttpRequest(); + + // run on data received back + xhr.onreadystatechange = function() { + if (this.readyState != 4) { + return; + } + if (this.status != 200) { + set_status("Server Error while retrieving filename, " + filename + ", status: " + this.status + " " + this.statusText); + return; + } + + var filename = this.responseText; + + get_file(filename); + } + + var parser = new DOMParser(); + var xmlDocument = parser.parseFromString("", "text/xml"); + + var requestElement = xmlDocument.getElementsByTagName("request")[0]; + + var commandElement = xmlDocument.createElement("command"); + commandElement.appendChild(document.createTextNode("getfilename")); + requestElement.appendChild(commandElement); + + var urlElement = xmlDocument.createElement("url"); + urlElement.appendChild(document.createTextNode(document.getElementById("url").value)); + requestElement.appendChild(urlElement); + + var formatElement = xmlDocument.createElement("format"); + formatElement.appendChild(document.createTextNode(document.getElementById("mp3").checked ? "mp3" : "mp4")); + requestElement.appendChild(formatElement); + + xhr.open("POST", "downtube.fcgi", true); + xhr.setRequestHeader("Content-type", "text/xml"); + xhr.responseType = 'text'; + xhr.send(xmlDocument); + + set_status("Please wait while retrieving filename..."); +} + +// started on button click: get file +function get_file(filename) { + var xhr = new XMLHttpRequest(); + + // run on data received back + xhr.onreadystatechange = function() { + if (this.readyState == 3) { + set_status("Please wait while downloading " + filename + " ..."); + return; + } + if (this.readyState != 4) { + return; + } + if (this.status != 200) { + set_status("Server Error while retrieving " + filename + ", status: " + this.status + " " + this.statusText); + return; + } + + var a = document.getElementById("download-a"); + a.setAttribute("download", filename); + var file = new Blob([this.response]); + a.href = window.URL.createObjectURL(file); + a.click(); + + set_status(""); // OK + } + + var parser = new DOMParser(); + var xmlDocument = parser.parseFromString("", "text/xml"); + + var requestElement = xmlDocument.getElementsByTagName("request")[0]; + + var commandElement = xmlDocument.createElement("command"); + commandElement.appendChild(document.createTextNode("getfile")); + requestElement.appendChild(commandElement); + + var urlElement = xmlDocument.createElement("url"); + urlElement.appendChild(document.createTextNode(document.getElementById("url").value)); + requestElement.appendChild(urlElement); + + var formatElement = xmlDocument.createElement("format"); + formatElement.appendChild(document.createTextNode(document.getElementById("mp3").checked ? "mp3" : "mp4")); + requestElement.appendChild(formatElement); + + xhr.open("POST", "downtube.fcgi", true); + xhr.setRequestHeader("Content-type", "text/xml"); + xhr.responseType = 'blob'; + xhr.send(xmlDocument); + + set_status("Please wait while server prepares " + filename + " ..."); +} + diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..b42c33e --- /dev/null +++ b/start.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# +# Test script for debugging +# +spawn-fcgi -a 127.0.0.1 -p 9014 -n -- ./whiteboard.fcgi diff --git a/webserver.conf.example b/webserver.conf.example new file mode 100644 index 0000000..eeb48a1 --- /dev/null +++ b/webserver.conf.example @@ -0,0 +1,8 @@ + + static-files + /usr/lib/whiteboard/html + + + fcgi + 127.0.0.1:9014 + diff --git a/whiteboard.cpp b/whiteboard.cpp new file mode 100644 index 0000000..60cfcc2 --- /dev/null +++ b/whiteboard.cpp @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "file.h" + +namespace pt = boost::property_tree; +using namespace std::string_literals; +namespace fs = std::filesystem; + +namespace { + + class TempDir + { + private: + fs::path m_path; + fs::path m_oldpath; + public: + TempDir() + { + char templ[] = "/tmp/downtubeXXXXXX"; + char *temp = mkdtemp(templ); + if (temp == nullptr) { + throw std::runtime_error("Can't create temporary directory."); + } + + m_path = temp; + m_oldpath = fs::current_path(); + fs::current_path(m_path); + } + + ~TempDir() + { + fs::current_path(m_oldpath); + fs::remove_all(m_path); + } + + const fs::path& getPath() const {return m_path;} + }; + + std::regex re{"https?://(www\\.youtube\\.com/watch\\?v=|youtu\\.be/)[[:alnum:]_&=-]+", std::regex::extended}; // www.youtube.com/watch?v=ItjMIxS3-rI or https://youtu.be/ItjMIxS3-rI + +} // anonymous namespace + +int main(void) +{ + int result = FCGX_Init(); + if (result != 0) { // error on init + fprintf(stderr, "Error: FCGX_Init()\n"); + return 1; + } + + result = FCGX_IsCGI(); + if (result) { + fprintf(stderr, "Error: No FCGI environment available.\n"); + return 1; + } + + FCGX_Request request; + result = FCGX_InitRequest(&request, 0, 0); + if (result != 0) { + fprintf(stderr, "Error: FCGX_InitRequest()\n"); + return 1; + } + + while (FCGX_Accept_r(&request) >= 0) { + try { + char* method = FCGX_GetParam("REQUEST_METHOD", request.envp); + + // POST for server actions, changes + if (!strcmp(method, "POST")) { + size_t contentLength { std::stoul(FCGX_GetParam("CONTENT_LENGTH", request.envp)) }; + std::string postData(contentLength, '\0'); // contentLength number of bytes, initialize with 0 + if (FCGX_GetStr(postData.data(), contentLength, request.in) != contentLength) { + throw std::runtime_error("Bad data read: Content length mismatch.\r\n"); + } + // postData contains POST data + std::string contentType(FCGX_GetParam("CONTENT_TYPE", request.envp)); + + std::string xmlData = postData; // default: interpret whole POST data as xml request + + pt::ptree xml; + std::istringstream ss{xmlData}; + pt::xml_parser::read_xml(ss, xml, pt::xml_parser::no_comments | pt::xml_parser::trim_whitespace); + + std::string url {xml.get("request.url")}; + std::string format {xml.get("request.format")}; + std::string command {xml.get("request.command")}; + + //FCGX_PutS("Content-Type: text/plain\r\n\r\n", request.out); + //FCGX_FPrintF(request.out, "url: %s\r\n", url.c_str()); + //FCGX_FPrintF(request.out, "format: %s\r\n", format.c_str()); // mp3, mp4 + + if (format != "mp3" && format != "mp4") { + throw std::runtime_error("Bad format: "s + format); + } + + if (!std::regex_match(url, re)) { + throw std::runtime_error("Bad URL"); + } + + // remove trailing "&..." + size_t and_pos {url.find('&')}; + if (and_pos != std::string::npos) + url = url.substr(0, and_pos); + + //FCGX_FPrintF(request.out, "command: %s\r\n", command.c_str()); + + TempDir tempDir; + + //FCGX_FPrintF(request.out, "path: %s\r\n", tempDir.getPath().string().c_str()); + + if (command == "getfilename") { + //std::string cmd{"youtube-dl -o '%(title)s."s + format + "' --get-filename --no-call-home --restrict-filenames "s + url + " > filename.txt"}; + // Recoding to MP4 is too slow currently. So keep original format for now + std::string cmd{"youtube-dl -o '%(title)s.%(ext)s' --get-filename --no-call-home --restrict-filenames "s + url + " > filename.txt"}; + if (system(cmd.c_str())) + throw std::runtime_error("Can't guess filename"); + + std::string filename {File::getFile("filename.txt")}; + boost::algorithm::trim(filename); + + if (format == "mp3") + filename = fs::path{filename}.stem().string() + ".mp3"; + + FCGX_PutS("Content-Type: text/plain\r\n\r\n", request.out); + FCGX_FPrintF(request.out, "%s", filename.c_str()); + } else if (command == "getfile") { + if (format == "mp3") { + std::string cmd{"youtube-dl --no-warnings --no-call-home --no-progress -x --audio-format mp3 -o 'audio.%(ext)s' --restrict-filenames "s + url}; + system(cmd.c_str()); // Ignore error - "ERROR: Stream #1:0 -> #0:1 (copy)" - seems to be ok + + std::string filedata {File::getFile("audio.mp3")}; // may throw + + if (filedata.size() > 30000000) + throw std::runtime_error("File too big"); + + FCGX_PutS("Content-Type: application/octet-stream\r\n", request.out); + FCGX_FPrintF(request.out, "Content-Length: %d\r\n\r\n", filedata.size()); + FCGX_PutStr(filedata.c_str(), filedata.size(), request.out); + } else if (format == "mp4") { + //std::string cmd{"youtube-dl --no-warnings --no-call-home --no-progress --recode-video mp4 -o video.mp4 --restrict-filenames "s + url}; + // Recoding to MP4 is too slow currently. So keep original format for now + std::string cmd{"youtube-dl --no-warnings --no-call-home --no-progress -o video.mp4 --restrict-filenames "s + url}; + system(cmd.c_str()); // Ignore error + + // youtube-dl gets it wrong and creates, e.g. video.mkv. + // So find it and load it + fs::directory_iterator di{fs::current_path()}; + fs::path filename; + for (const auto& i: di) { + if (boost::algorithm::starts_with(i.path().filename().string(), "video."s)) { + filename = i.path().filename().string(); + break; + } + } + + if (filename.empty()) { + throw std::runtime_error("No video file found."); + } + + std::string filedata {File::getFile(filename)}; // may throw + + if (filedata.size() > 300000000) + throw std::runtime_error("File too big"); + + FCGX_PutS("Content-Type: application/octet-stream\r\n", request.out); + FCGX_FPrintF(request.out, "Content-Length: %d\r\n\r\n", filedata.size()); + FCGX_PutStr(filedata.c_str(), filedata.size(), request.out); + } else { + throw std::runtime_error("Bad format for unknown reason: "s + format); // should have been caught above already! + } + } else { + throw std::runtime_error("Bad command: "s + command); + } + + // Name: + // youtube-dl -o "%(title)s.mp3" --get-filename --no-call-home --restrict-filenames $SOURCE > filename.txt + // + // MP3: + // youtube-dl --no-warnings --no-call-home --no-progress -x --audio-format mp3 --embed-thumbnail -o "%(title)s.(ext)s" --restrict-filenames $SOURCE + // + // MP4: + // youtube-dl --no-warnings --no-call-home --no-progress --recode-video mp4 -o "%(title)s.(ext)s" --restrict-filenames $SOURCE + } else { + throw std::runtime_error("Unsupported method.\r\n"); + } + } catch (const std::runtime_error& ex) { + FCGX_PutS("Status: 500 Internal Server Error\r\n", request.out); + FCGX_PutS("Content-Type: text/html\r\n\r\n", request.out); + FCGX_FPrintF(request.out, "Error: %s\r\n", ex.what()); + } catch (const std::exception& ex) { + FCGX_PutS("Status: 500 Internal Server Error\r\n", request.out); + FCGX_PutS("Content-Type: text/html\r\n\r\n", request.out); + FCGX_FPrintF(request.out, "Unknown exception: %s\r\n", ex.what()); + } + } + + return 0; +} + -- cgit v1.2.3