From 2ef9f51df48b14556e236d14213233e1bd7f829a Mon Sep 17 00:00:00 2001 From: Roland Reichwein Date: Sat, 30 Jan 2021 19:20:15 +0100 Subject: Added Support for Debian and Ubuntu, add is_valid_utf() --- Makefile | 20 ++++++++++++++++++-- debian/control | 3 +++ include/unicode.h | 11 +++++++++++ src/test-unicode.cpp | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 46626e8..b66c17e 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,38 @@ PROJECTNAME=unicode VERSION=$(shell dpkg-parsechangelog --show-field Version) -DISTROS=base #debian10 +DISTROS=base debian10 ubuntu2004 ubuntu2010 +ifeq ($(wildcard $(shell which clang++-11)),) +ifeq ($(wildcard $(shell which clang++)),) +$(error No clang++-11 nor clang++ available!) +else +CXX=clang++ +endif +else CXX=clang++-11 # GCC is buggy: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85282 #CXX=g++-10 +endif STANDARD=c++17 #STANDARD=c++20 +ifeq ($(CXXFLAGS),) CXXFLAGS=-O0 -g -D_DEBUG #CXXFLAGS=-O2 -DNDEBUG +endif CXXFLAGS+=-Wall -Iinclude -std=$(STANDARD) ifeq ($(CXX),clang++-11) COMPILER_SUITE=clang +LIBS+=-fuse-ld=lld-11 +endif + +ifeq ($(CXX),clang++) +COMPILER_SUITE=clang +LIBS+=-fuse-ld=lld endif ifeq ($(COMPILER_SUITE),clang) @@ -31,7 +47,6 @@ LDLIBS+=\ ifeq ($(COMPILER_SUITE),clang) LIBS+= \ --fuse-ld=lld-11 \ -lc++ \ -lc++abi #-lc++fs @@ -68,6 +83,7 @@ dep: $(SRC:.cpp=.d) clean: -rm -f src/recode src/test-unicode + -rm -rf result -find . -name '*.o' -o -name '*.d' -o -name '*.gcno' -o -name '*.gcda' | xargs rm -f install: diff --git a/debian/control b/debian/control index 42b6c22..1572512 100644 --- a/debian/control +++ b/debian/control @@ -16,3 +16,6 @@ Description: Unicode conversion library . Features: - Additional support for ISO-8859-1 encoding (Latin-1) as subset of Unicode + - Additional support for ISO-8859-15 + - Tested on Debian 10, Ubuntu 2004, Ubuntu 2010 + - C++17 and C++20 compatible diff --git a/include/unicode.h b/include/unicode.h index 9e0132b..f31cbac 100644 --- a/include/unicode.h +++ b/include/unicode.h @@ -339,5 +339,16 @@ namespace unicode { return result; } + template + bool is_valid_utf(const std::basic_string& s) + { + try { + std::for_each(utf_begin(s), utf_end(s), [](const T& c){}); + } catch(...) { + return false; + } + return true; + } + } // namespace unicode diff --git a/src/test-unicode.cpp b/src/test-unicode.cpp index 05370c7..3d67124 100644 --- a/src/test-unicode.cpp +++ b/src/test-unicode.cpp @@ -37,7 +37,7 @@ std::vector success_sets { std::vector> failure_strings_char8_t { u8"\x80", // utf-8 continuation byte u8"\x81", // utf-8 continuation byte - u8"\xc3ä", // initial byte of utf-8 "ä", followed by valid utf-8 "ä" + u8"\xc3\xc3\xa4", // initial byte of utf-8 "ä", followed by valid utf-8 "ä" u8"\xF8\x80\x80\x80\x80", // overlong encoding u8"\xF7\xBF\xBF\xBF", // valid encoding of invalid code point }; @@ -56,6 +56,7 @@ std::vector> failure_strings_char32_t { // output operators must be in same namespace as the type itself namespace std { +#ifdef __cpp_char8_t std::ostream& operator<<(std::ostream& os, std::basic_string const& s) { os << "["; @@ -65,6 +66,7 @@ std::ostream& operator<<(std::ostream& os, std::basic_string const& s) return os; } +#endif std::ostream& operator<<(std::ostream& os, std::basic_string const& s) { @@ -118,6 +120,27 @@ BOOST_AUTO_TEST_CASE(utf_to_utf_success) test_utf_to_utf(t); } +template +void test_is_valid_utf(std::tuple& t) +{ + typedef typename std::tuple_element::type>::type T; + + // test + bool result { unicode::is_valid_utf(std::get(t)) }; + + BOOST_CHECK_MESSAGE(result == true, "is_valid_utf w/ " << typeid(T).name() << "(" << i << ", " << std::get(t) << "), got " << result); + + // iterate over other combinations + if constexpr (i + 1 < std::tuple_size::type>::value) + test_is_valid_utf(t); +} + +BOOST_AUTO_TEST_CASE(is_valid_utf_success) +{ + for (auto& t: success_sets) + test_is_valid_utf(t); +} + // iterate over std::tuple T types template void test_utf_to_utf_failure(std::basic_string& s) @@ -126,7 +149,7 @@ void test_utf_to_utf_failure(std::basic_string& s) try { unicode::utf_to_utf(s); - BOOST_FAIL("Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name()); + BOOST_ERROR("Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name()); } catch (...) { // OK }; @@ -148,6 +171,29 @@ BOOST_AUTO_TEST_CASE(utf_to_utf_failure) test_utf_to_utf_failure::type::value_type, types_collection_type>(s); } +// iterate over std::tuple T types +template +void test_is_valid_utf_failure(std::basic_string& s) +{ + BOOST_CHECK_MESSAGE(unicode::is_valid_utf(s) == false, "Expected bad UTF at index: " << index << ", " << typeid(T).name()); + + // iterate over remaining types + if constexpr (index + 1 < std::tuple_size::value) + test_is_valid_utf_failure(s); +} + +BOOST_AUTO_TEST_CASE(is_valid_utf_failure) +{ + for (auto& s: failure_strings_char8_t) + test_is_valid_utf_failure::type::value_type, types_collection_type>(s); + + for (auto& s: failure_strings_char16_t) + test_is_valid_utf_failure::type::value_type, types_collection_type>(s); + + for (auto& s: failure_strings_char32_t) + test_is_valid_utf_failure::type::value_type, types_collection_type>(s); +} + BOOST_AUTO_TEST_CASE(is_valid_unicode) { BOOST_CHECK(unicode::is_valid_unicode('\0')); -- cgit v1.2.3