path: root/src/test-unicode.cpp

#define BOOST_TEST_MODULE unicode_test

#include <boost/test/included/unit_test.hpp>
#include <boost/test/data/dataset.hpp>
#include <boost/test/data/monomorphic.hpp>
#include <boost/test/data/test_case.hpp>

#include <array>
#include <chrono>
#include <deque>
#include <exception>
#include <limits>
#include <list>
#include <random>
#include <string>
#include <tuple>
#include <type_traits>
#include <vector>

#include <unicode.h>

using namespace std::chrono_literals;

typedef std::tuple<std::basic_string<utf8_t>, std::basic_string<char16_t>, std::basic_string<char32_t>> types_collection_type;

// create tuple of the same string, in UTF-8, UTF-16 and UTF-32
#define SUCCESS_TUPLE(x) {u8 ## x, u ## x, U ## x}

// Success cases: convert string to all other types, respectively
std::vector<types_collection_type> success_sets {
 SUCCESS_TUPLE(""),
 SUCCESS_TUPLE("ASCII string1"),
 SUCCESS_TUPLE("Täst just looks like German"),
 SUCCESS_TUPLE("\u732b is chinese for cat"),
 SUCCESS_TUPLE("\U0001F63A"),
 SUCCESS_TUPLE("\U0001F63A is a smiling cat"),
};

// Error cases: throwing upon convert to all other types
std::vector<std::basic_string<utf8_t>> failure_strings_char8_t {
 // using u8"" here doesn't work on MSVC
 (utf8_t*)"\x80", // utf-8 continuation byte
 (utf8_t*)"\x81", // utf-8 continuation byte
 (utf8_t*)"\xc3\xc3\xa4", // initial byte of utf-8 "ä", followed by valid utf-8 "ä"
 (utf8_t*)"\xF8\x80\x80\x80\x80", // overlong encoding
 (utf8_t*)"\xF7\xBF\xBF\xBF", // valid encoding of invalid code point
};

std::vector<std::basic_string<char16_t>> failure_strings_char16_t {
 u"\xD801", // single high surrogate
 u"\xDFFF", // single low surrogate
 u"\xDFFF\xD801", // bad surrogate pair order
};

std::vector<std::basic_string<char32_t>> failure_strings_char32_t {
 U"\xD800 and more text", // invalid unicode (surrogate half)
 U"blabla \xD801", // invalid unicode (surrogate half)
 U"moreblabla \xDFFF", // invalid unicode (surrogate half)
 U"\x10000000", // invalid unicode (number too big)
};

// output operators must be in same namespace as the type itself
namespace std {

#ifdef __cpp_char8_t
std::ostream& operator<<(std::ostream& os, std::basic_string<utf8_t> const& s)
{
 os << "[";
 for (auto& c: s)
  os << " " << std::to_string(static_cast<uint8_t>(c));
 os << "]";

 return os;
}
#endif

std::ostream& operator<<(std::ostream& os, std::basic_string<char16_t> const& s)
{
 os << "[";
 for (auto& c: s)
  os << " " << std::to_string(static_cast<uint16_t>(c));
 os << "]";

 return os;
}

std::ostream& operator<<(std::ostream& os, std::basic_string<char32_t> const& s)
{
 os << "[";
 for (auto& c: s)
  os << " " << std::to_string(static_cast<uint32_t>(c));
 os << "]";

 return os;
}

}

template<size_t i = 0, size_t j = 0, typename... Ts>
void test_utf_to_utf(std::tuple<Ts...>& t)
{
 typedef typename std::tuple_element<i,typename std::remove_reference<decltype(t)>::type>::type From;
 typedef typename std::tuple_element<j,typename std::remove_reference<decltype(t)>::type>::type To;

 // test base type interface
 To result { unicode::convert<typename From::value_type, typename To::value_type>(std::get<i>(t)) };
 BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "Base: From " << typeid(typename From::value_type).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(typename To::value_type).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);

 // test container interface
 result = unicode::convert<From, To>(std::get<i>(t));
 BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "Container: From " << typeid(From).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(To).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);

 // test facet interface
 result = unicode::convert<typename unicode::Encoding<typename From::value_type>::Facet, typename unicode::Encoding<typename To::value_type>::Facet>(std::get<i>(t));
 BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "Facet: From " << typeid(From).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(To).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);
 
 // iterate over other combinations
 if constexpr (i + 1 < std::tuple_size<typename std::remove_reference<decltype(t)>::type>::value)
  test_utf_to_utf<i + 1, j>(t);
 else if constexpr (j + 1 < std::tuple_size<typename std::remove_reference<decltype(t)>::type>::value)
  test_utf_to_utf<0, j + 1>(t);
}

// We don't use BOOST_DATA_TEST_CASE here because boost::test tries to assign
// a new variable to each tuple element which we don't want
// https://lists.boost.org/boost-bugs/2016/05/45214.php

BOOST_AUTO_TEST_CASE(utf_to_utf_success)
{
 for (auto& t: success_sets)
  test_utf_to_utf(t);
}

template<size_t i = 0, typename... Ts>
void test_is_valid_utf(std::tuple<Ts...>& t)
{
 typedef typename std::tuple_element<i,typename std::remove_reference<decltype(t)>::type>::type T;

 // test via basic type
 bool result { unicode::is_valid_utf<typename T::value_type>(std::get<i>(t)) };
 BOOST_CHECK_MESSAGE(result == true, "is_valid_utf w/ " << typeid(typename T::value_type).name() << "(" << i << ", " << std::get<i>(t) << "), got " << result);

 // test via container type
 result = unicode::is_valid_utf<T>(std::get<i>(t));
 BOOST_CHECK_MESSAGE(result == true, "is_valid_utf w/ " << typeid(T).name() << "(" << i << ", " << std::get<i>(t) << "), got " << result);

 // test via Facet
 result = unicode::is_valid_utf<typename unicode::Encoding<typename T::value_type>::Facet>(std::get<i>(t));
 BOOST_CHECK_MESSAGE(result == true, "is_valid_utf w/ " << typeid(typename unicode::Encoding<typename T::value_type>::Facet).name() << "(" << i << ", " << std::get<i>(t) << "), got " << result);

 // iterate over other combinations
 if constexpr (i + 1 < std::tuple_size<typename std::remove_reference<decltype(t)>::type>::value)
  test_is_valid_utf<i + 1>(t);
}

BOOST_AUTO_TEST_CASE(is_valid_utf_success)
{
 for (auto& t: success_sets)
  test_is_valid_utf(t);
}

// iterate over std::tuple T types
template<typename From, typename Collection, size_t index = 0>
void test_utf_to_utf_failure(std::basic_string<From>& s)
{
 typedef typename std::tuple_element<index, Collection>::type::value_type To;

 // via base type
 try {
  (void) unicode::convert<From,To>(s);
  BOOST_ERROR("Base type: Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 };

 // via container
 try {
  (void) unicode::convert<typename unicode::Encoding<From>::Facet::string_type, typename unicode::Encoding<To>::Facet::string_type>(s);
  BOOST_ERROR("Container type: Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 };

 // via facet
 try {
  (void) unicode::convert<typename unicode::Encoding<From>::Facet,typename unicode::Encoding<To>::Facet>(s);
  BOOST_ERROR("Facet: Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 };

 // iterate over remaining types 
 if constexpr (index + 1 < std::tuple_size<Collection>::value)
  test_utf_to_utf_failure<From, Collection, index + 1>(s);
}

BOOST_AUTO_TEST_CASE(utf_to_utf_failure)
{
 for (auto& s: failure_strings_char8_t)
  test_utf_to_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);
 
 for (auto& s: failure_strings_char16_t)
  test_utf_to_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);

 for (auto& s: failure_strings_char32_t)
  test_utf_to_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);
}

// iterate over std::tuple T types
template<typename T, typename Collection, size_t index = 0>
void test_is_valid_utf_failure(std::basic_string<T>& s)
{
 BOOST_CHECK_MESSAGE(unicode::is_valid_utf<T>(s) == false, "Expected bad UTF at index: " << index << ", " << typeid(T).name());
 
 BOOST_CHECK_MESSAGE(unicode::is_valid_utf<typename std::basic_string<T>>(s) == false, "Expected bad UTF at index: " << index << ", " << typeid(T).name());
 
 BOOST_CHECK_MESSAGE(unicode::is_valid_utf<typename unicode::Encoding<T>::Facet>(s) == false, "Expected bad UTF at index: " << index << ", " << typeid(typename unicode::Encoding<T>::Facet).name());

 // iterate over remaining types 
 if constexpr (index + 1 < std::tuple_size<Collection>::value)
  test_is_valid_utf_failure<T, Collection, index + 1>(s);
}

BOOST_AUTO_TEST_CASE(is_valid_utf_failure)
{
 for (auto& s: failure_strings_char8_t)
  test_is_valid_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);
 
 for (auto& s: failure_strings_char16_t)
  test_is_valid_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);

 for (auto& s: failure_strings_char32_t)
  test_is_valid_utf_failure<typename std::remove_reference<decltype(s)>::type::value_type, types_collection_type>(s);
}

BOOST_AUTO_TEST_CASE(is_valid_unicode)
{
 BOOST_CHECK(unicode::is_valid_unicode('\0'));
 BOOST_CHECK(unicode::is_valid_unicode(U'a'));
 BOOST_CHECK(unicode::is_valid_unicode(U'ä'));
 BOOST_CHECK(unicode::is_valid_unicode(U'\u732b')); // cat chinese
 BOOST_CHECK(unicode::is_valid_unicode(U'\U0001F63A')); // cat chinese
 BOOST_CHECK(unicode::is_valid_unicode(0x0001F63A)); // cat smiley

 BOOST_CHECK(!unicode::is_valid_unicode(0x00110000));
 BOOST_CHECK(!unicode::is_valid_unicode(0xFFFFFFFF)); // U"\UFFFFFFFF" is invalid C++
 BOOST_CHECK(!unicode::is_valid_unicode(0x01234567));
 BOOST_CHECK(!unicode::is_valid_unicode(0x12345678));
 BOOST_CHECK(!unicode::is_valid_unicode(0xD800));
 BOOST_CHECK(!unicode::is_valid_unicode(0xD987));
 BOOST_CHECK(!unicode::is_valid_unicode(0xDFFF));
}

struct random_context {
 std::random_device rd;  // OS random number engine to seed RNG (below)
 std::mt19937 gen{rd()};
 std::uniform_int_distribution<size_t> sequence_length{0, 100000}; // length of sequence: 0 ... 100000 code units
 std::uniform_int_distribution<unsigned long> code_point_distribution{0, 0x10FFFF - 0x800};
};

// generates valid and invalid strings of different type
template<typename T>
T generate_random_invalid(random_context& rc, size_t length)
{
 // Using unsigned long for std::uniform_int_distribution<> because it needs to be basic type according to MSVC
 std::uniform_int_distribution<unsigned long> code_unit{0, std::numeric_limits<typename T::value_type>::max()}; // code unit value
 T result;
 std::generate_n(std::back_inserter(result), length, [&](){return static_cast<typename T::value_type>(code_unit(rc.gen));});

 return result;
}

char32_t generate_random_char(random_context& rc)
{
 auto result {rc.code_point_distribution(rc.gen)};
 if (result >= 0xD800)
  result += 0x800;
 return static_cast<char32_t>(result);
}

std::u32string generate_random_string(random_context& rc, size_t length)
{
 std::u32string result;
 std::generate_n(std::back_inserter(result), length, [&](){return generate_random_char(rc);});

 return result;
}

template<typename From, typename ToTypesCollectionType, size_t i = 0>
void test_random(random_context& rc, size_t length)
{
 //std::cerr << "LENGTH: " << length << std::endl;
 typedef typename std::tuple_element<i,ToTypesCollectionType>::type To;

 From r {static_cast<From>(generate_random_invalid<From>(rc, length))};

 // base type interface
 try {
  To result{unicode::convert<typename From::value_type,typename To::value_type>(r)};

  if (r.empty()) {
   BOOST_CHECK(result.empty());
  } else {
   BOOST_CHECK(!result.empty());
  }
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 }

 // container type interface
 try {
  To result{unicode::convert<From, To>(r)};

  if (r.empty()) {
   BOOST_CHECK(result.empty());
  } else {
   BOOST_CHECK(!result.empty());
  }
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 }

 // facet interface
 try {
  To result{unicode::convert<typename unicode::Encoding<typename From::value_type>::Facet,typename unicode::Encoding<typename To::value_type>::Facet>(r)};

  if (r.empty()) {
   BOOST_CHECK(result.empty());
  } else {
   BOOST_CHECK(!result.empty());
  }
 } catch (const std::invalid_argument&) {
  // OK: this is an expected exception for convert() on bad input
 } catch (const std::exception& ex) {
  BOOST_ERROR("Unexpected error on convert(): " << ex.what());
 }

 // iterate over remaining To types
 if constexpr (i + 1 < std::tuple_size<ToTypesCollectionType>::value)
  test_random<From, ToTypesCollectionType, i + 1>(rc, length);
}

BOOST_AUTO_TEST_CASE_TEMPLATE(random_sequences_invalid, T, types_collection_type)
{
 random_context rc;

 for (int i = 0; i < 10; i++) {
  test_random<T,types_collection_type>(rc, rc.sequence_length(rc.gen));
 }
}

BOOST_AUTO_TEST_CASE(random_sequences_valid)
{
 random_context rc;

 // Fill UTF-32 data list
 std::vector<std::u32string> u32list;
 std::generate_n(std::back_inserter(u32list), 1000, [&](){return generate_random_string(rc, rc.sequence_length(rc.gen));});

 // Fill UTF-16 data list
 std::vector<std::u16string> u16list;
 std::transform(u32list.begin(), u32list.end(), std::back_inserter(u16list), [](const std::u32string& s){return unicode::convert<unicode::UTF_32, unicode::UTF_16>(s);});

 // Fill UTF-8 data list
 std::vector<std::basic_string<utf8_t>> u8list;
 std::transform(u32list.begin(), u32list.end(), std::back_inserter(u8list), [](const std::u32string& s){return unicode::convert<unicode::UTF_32, unicode::UTF_8>(s);});

 for (const auto& i : u32list) {
  std::u32string s32{unicode::convert<unicode::UTF_32, unicode::UTF_32>(i)};
  BOOST_CHECK(s32.size() == i.size());
  std::u16string s16{unicode::convert<unicode::UTF_32, unicode::UTF_16>(i)};
  BOOST_CHECK(s16.size() >= i.size());
  std::basic_string<utf8_t> s8{unicode::convert<unicode::UTF_32, unicode::UTF_8>(i)};
  BOOST_CHECK(s8.size() >= i.size());
 }

 for (const auto& i : u16list) {
  std::u32string s32{unicode::convert<unicode::UTF_16, unicode::UTF_32>(i)};
  BOOST_CHECK(s32.size() > 0 || i.size() == 0);
  std::u16string s16{unicode::convert<unicode::UTF_16, unicode::UTF_16>(i)};
  BOOST_CHECK(s16.size() == i.size());
  std::basic_string<utf8_t> s8{unicode::convert<unicode::UTF_16, unicode::UTF_8>(i)};
  BOOST_CHECK(s8.size() >= i.size());
 }

 for (const auto& i : u8list) {
  std::u32string s32{unicode::convert<unicode::UTF_8, unicode::UTF_32>(i)};
  BOOST_CHECK(s32.size() > 0 || i.size() == 0);
  std::u16string s16{unicode::convert<unicode::UTF_8, unicode::UTF_16>(i)};
  BOOST_CHECK(s16.size() > 0 || i.size() == 0);
  std::basic_string<utf8_t> s8{unicode::convert<unicode::UTF_8, unicode::UTF_8>(i)};
  BOOST_CHECK(s8.size() == i.size());
 }

 {
  // Performance test UTF-32 -> UTF-32
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u32list) {
   std::u32string s{unicode::convert<unicode::UTF_32, unicode::UTF_32>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-32 to UTF-32: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-32 -> UTF-16
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u32list) {
   std::u16string s{unicode::convert<unicode::UTF_32, unicode::UTF_16>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-32 to UTF-16: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-32 -> UTF-8
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u32list) {
   std::basic_string<utf8_t> s{unicode::convert<unicode::UTF_32, unicode::UTF_8>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-32 to UTF-8: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-16 -> UTF-32
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u16list) {
   std::u32string s{unicode::convert<unicode::UTF_16, unicode::UTF_32>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-16 to UTF-32: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-16 -> UTF-16
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u16list) {
   std::u16string s{unicode::convert<unicode::UTF_16, unicode::UTF_16>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-16 to UTF-16: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-16 -> UTF-8
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u16list) {
   std::basic_string<utf8_t> s{unicode::convert<unicode::UTF_16, unicode::UTF_8>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-16 to UTF-8: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-8 -> UTF-32
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u8list) {
   std::u32string s{unicode::convert<unicode::UTF_8, unicode::UTF_32>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-8 to UTF-32: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-8 -> UTF-16
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u8list) {
   std::u16string s{unicode::convert<unicode::UTF_8, unicode::UTF_16>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-8 to UTF-16: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

 {
  // Performance test UTF-8 -> UTF-8
  auto t0{std::chrono::steady_clock::now()};
  for (const auto& i : u8list) {
   std::basic_string<utf8_t> s{unicode::convert<unicode::UTF_8, unicode::UTF_8>(i)};
  }
  std::cout << "Performance test for converting 1M strings from UTF-8 to UTF-8: " << std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() << std::endl;
 }

}

// Test ISO and UTF encodings
BOOST_AUTO_TEST_CASE(convert)
{
 BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>({})}) == std::string{});
 BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("abc")}) == std::string{"abc"});
 BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("äöü")}) == std::string{"äöü"});
 BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("\xa4")}) == std::string{"\xa4"}); // €
 
 BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_15,unicode::ISO_8859_15>("\xa4")}) == std::string{"\xa4"}); // €
 
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::ISO_8859_15,unicode::ISO_8859_1>("\xa4")}), std::invalid_argument); // € not available in ISO-8859-1
 
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_8,unicode::ISO_8859_1>(u8"\u20ac")}), std::invalid_argument);
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_16,unicode::ISO_8859_1>(u"\u20ac")}), std::invalid_argument);
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_32,unicode::ISO_8859_1>(U"\u20ac")}), std::invalid_argument);
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_8,unicode::ISO_8859_15>(u8"\u732b")}), std::invalid_argument);
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_16,unicode::ISO_8859_15>(u"\u732b")}), std::invalid_argument);
 BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::UTF_32,unicode::ISO_8859_15>(U"\u732b")}), std::invalid_argument);

 BOOST_CHECK_THROW((unicode::convert<unicode::UTF_32,unicode::UTF_8>(std::u32string{(char32_t*)"\x00\xD8\x00\x00\x00\x00\x00\x00"})) , std::invalid_argument);

 BOOST_CHECK((unicode::convert<unicode::UTF_8,unicode::UTF_16>(u8"abc")) == std::u16string{u"abc"});
 BOOST_CHECK((unicode::convert<unicode::UTF_32,unicode::UTF_16>(U"abc")) == std::u16string{u"abc"});

 BOOST_CHECK((unicode::convert<utf8_t,char16_t>(u8"abc")) == std::u16string{u"abc"});
 BOOST_CHECK((unicode::convert<char32_t,char16_t>(U"abc")) == std::u16string{u"abc"});

 BOOST_CHECK((unicode::convert<char, char32_t>("äöü")) == std::u32string{U"äöü"});

#ifdef _WIN32
 BOOST_CHECK(sizeof(wchar_t) == 2);
#else // Unix like
 BOOST_CHECK(sizeof(wchar_t) == 4);
#endif

 // For the following checks, wchar_t size and encoding is system dependent:
 // Windows: UTF-16
 // Linux: UTF-32
 BOOST_CHECK((unicode::convert<char, wchar_t>("äöü")) == std::wstring{L"äöü"});
 BOOST_CHECK((unicode::convert<char, wchar_t>("\u732b")) == std::wstring{L"\u732b"});
 BOOST_CHECK((unicode::convert<char, wchar_t>("\U0001F63A")) == std::wstring{L"\U0001F63A"});
 BOOST_CHECK((unicode::convert<wchar_t, char32_t>(L"\U0001F63A")) == std::u32string{U"\U0001F63A"});
 BOOST_CHECK((unicode::convert<wchar_t, utf8_t>(L"\U0001F63A")) == std::basic_string<utf8_t>{(utf8_t*)"\U0001F63A"});

 BOOST_CHECK((unicode::convert<std::string, std::wstring>(std::string{"äöü"})) == std::wstring{L"äöü"});
 
 BOOST_CHECK((unicode::convert<std::vector<char>, std::vector<wchar_t>>(std::vector<char>{})) == std::vector<wchar_t>{});
 BOOST_CHECK((unicode::convert<std::vector<char>, std::vector<wchar_t>>(std::vector<char>{'\xc3', '\xa4', '\xc3', '\xb6', '\xc3', '\xbc'})) == (std::vector<wchar_t>{L'ä', L'ö', L'ü'}));
 
 // deque
 BOOST_CHECK((unicode::convert<std::deque<char>, std::deque<wchar_t>>(std::deque<char>{})) == std::deque<wchar_t>{});
 BOOST_CHECK((unicode::convert<std::deque<char>, std::deque<wchar_t>>(std::deque<char>{'\xc3', '\xa4', '\xc3', '\xb6', '\xc3', '\xbc'})) == (std::deque<wchar_t>{L'ä', L'ö', L'ü'}));
 
 // deque with uint8_t, uint16_t
 BOOST_CHECK((unicode::convert<std::deque<uint8_t>, std::deque<uint16_t>>(std::deque<uint8_t>{})) == std::deque<uint16_t>{});
 BOOST_CHECK((unicode::convert<std::deque<uint8_t>, std::deque<uint16_t>>(std::deque<uint8_t>{0xc3, 0xa4, 0xc3, 0xb6, 0xc3, 0xbc})) == (std::deque<uint16_t>{L'ä', L'ö', L'ü'}));
 
 // deque with int8_t, int16_t
 BOOST_CHECK((unicode::convert<std::deque<int8_t>, std::deque<int16_t>>(std::deque<int8_t>{
                                                                        static_cast<int8_t>(0xc3),
                                                                        static_cast<int8_t>(0xa4),
                                                                        static_cast<int8_t>(0xc3),
                                                                        static_cast<int8_t>(0xb6),
                                                                        static_cast<int8_t>(0xc3),
                                                                        static_cast<int8_t>(0xbc)})) == (std::deque<int16_t>{L'ä', L'ö', L'ü'}));
 
 // list
 BOOST_CHECK((unicode::convert<std::list<uint8_t>, std::list<uint16_t>>(std::list<uint8_t>{})) == std::list<uint16_t>{});
 BOOST_CHECK((unicode::convert<std::list<uint8_t>, std::list<uint16_t>>(std::list<uint8_t>{0xc3, 0xa4, 0xc3, 0xb6, 0xc3, 0xbc})) == (std::list<uint16_t>{L'ä', L'ö', L'ü'}));

 // list -> deque
 BOOST_CHECK((unicode::convert<std::list<uint8_t>, std::deque<uint16_t>>(std::list<uint8_t>{})) == std::deque<uint16_t>{});
 BOOST_CHECK((unicode::convert<std::list<uint8_t>, std::deque<uint16_t>>(std::list<uint8_t>{0xc3, 0xa4, 0xc3, 0xb6, 0xc3, 0xbc})) == (std::deque<uint16_t>{L'ä', L'ö', L'ü'}));
 
 // array
 BOOST_CHECK((unicode::convert<std::array<uint8_t, 0>, std::list<uint16_t>>(std::array<uint8_t, 0>{})) == std::list<uint16_t>{});
 BOOST_CHECK((unicode::convert<std::array<uint8_t, 6>, std::list<uint16_t>>(std::array<uint8_t, 6>{0xc3, 0xa4, 0xc3, 0xb6, 0xc3, 0xbc})) == (std::list<uint16_t>{L'ä', L'ö', L'ü'}));
}

BOOST_AUTO_TEST_CASE(is_valid_utf)
{
 BOOST_CHECK(unicode::is_valid_utf<char16_t>(u"äöü"));

 BOOST_CHECK(unicode::is_valid_utf<unicode::UTF_8>(u8"äöü"));
}

// check assumptions about environment
BOOST_AUTO_TEST_CASE(string_u8string)
{
 std::string a{"\xc3\xa4"};

 std::basic_string<utf8_t> b{a.begin(), a.end()};

 BOOST_CHECK(b == std::basic_string<utf8_t>{u8"ä"});
 
 a = std::string{b.begin(), b.end()};
 
 BOOST_CHECK(a == std::string{"\xc3\xa4"});
}