path: root/include/unicode.h

// libunicode

#pragma once

#include <algorithm>
#include <iterator>
#include <memory>
#include <stdexcept>
#include <string>
#include <unordered_map>

#ifdef __cpp_char8_t
// char8_t available
 typedef char8_t utf8_t;
#else
 typedef char utf8_t;
#endif

namespace unicode {

 // usually, char32_t, uint32_t etc.
 template<typename T>
 static inline bool is_valid_unicode(const T& value)
 {
   return value <= 0x10FFFF && (value <= 0xD7FF || value >= 0xE000);
 }

}

namespace unicode::detail {

 using namespace std::string_literals;

 template<typename T>
 struct utf_iterator
 {
  typedef T input_type;
  typedef char32_t value_type;
  typedef char32_t& reference;
  typedef char32_t* pointer;
  typedef size_t difference_type;
  typedef std::input_iterator_tag iterator_category;
  typedef std::basic_string<T> string_type;

  utf_iterator(const typename string_type::const_iterator& cbegin, const typename string_type::const_iterator& cend):
   iterator(cbegin), end_iterator(cend)
  {
   calculate_value<T>();
  }

  utf_iterator<T>(const utf_iterator<T>& other) = default;
  utf_iterator<T>& operator=(const utf_iterator<T>& other) = default;

  size_t remaining_code_units()
  {
   return end_iterator - iterator;
  }

  template<size_t index>
  T get_code_unit()
  {
   return *(iterator + index);
  }

  // set value member
  // default: char32_t for UTF-32
  // specializations for UTF-8 and UTF-16 below
  template<typename T1>
  void calculate_value()
  {
   static_assert(sizeof(T1) == 4);

   size_t remaining{remaining_code_units()};

   if (!remaining)
    return;

   value = get_code_unit<0>();
   
   if (!unicode::is_valid_unicode(value))
    throw std::invalid_argument("Invalid Unicode character: "s + std::to_string(static_cast<uint32_t>(value)));

   sequence_length = 1;
  }

  inline static bool is_continuation_byte(T b)
  {
   return (b & 0b11000000) == 0b10000000;
  }

  template<typename... Targs>
  inline static bool is_continuation_byte(T b, Targs... Fargs)
  {
   return is_continuation_byte(b) && is_continuation_byte(Fargs...);
  }

  template<size_t n>
  inline static bool is_byte0_of(T b)
  {
   return (b & static_cast<T>(0xFF << (7 - n))) == static_cast<T>(0xFF << (8 - n));
  }

  inline static char32_t continuation_value(T b)
  {
   return static_cast<char32_t>(b & 0b00111111);
  }

  template<typename... Targs>
  inline static char32_t continuation_value(T b, Targs... Fargs)
  {
   return continuation_value(b) << (6 * sizeof...(Targs)) | continuation_value(Fargs...);
  }

  template<size_t n>
  inline static char32_t value_byte0_of(T b)
  {
   return static_cast<char32_t>(b & (0b1111111 >> n)) << ((n - 1) * 6);
  }

  // GCC Bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85282
  // specialization for UTF-8
  template<>
  void calculate_value<utf8_t>()
  {
   size_t remaining{remaining_code_units()};
   
   if (!remaining)
    return;

   utf8_t byte0 {get_code_unit<0>()};
   if (byte0 & 0x80) { // 2-4 bytes
    if (remaining >= 2) {
     utf8_t byte1 {get_code_unit<1>()};
     if (is_byte0_of<2>(byte0) && is_continuation_byte(byte1)) { // 2 bytes
      value = value_byte0_of<2>(byte0) | continuation_value(byte1);
      sequence_length = 2;
     } else if (remaining >= 3) {
      utf8_t byte2 {get_code_unit<2>()};
      if (is_byte0_of<3>(byte0) && is_continuation_byte(byte1, byte2)) { // 3 bytes
       value = value_byte0_of<3>(byte0) | continuation_value(byte1, byte2);
       sequence_length = 3;
      } else if (remaining >= 4) {
       utf8_t byte3 {get_code_unit<3>()};
       if (is_byte0_of<4>(byte0) && is_continuation_byte(byte1, byte2, byte3)) { // 4 bytes
        value = value_byte0_of<4>(byte0) | continuation_value(byte1, byte2, byte3);
        sequence_length = 4;
       } else
        throw std::invalid_argument("Bad input: Invalid 4 byte sequence");
      } else
       throw std::invalid_argument("Bad input: Invalid 3 byte sequence");
     } else
      throw std::invalid_argument("Bad input: Invalid 2 byte sequence");
    } else
     throw std::invalid_argument("Bad input: 2nd byte expected, none found");
  
    // check only for sequences >= 2 bytes (ASCII is always compliant)
    if (!unicode::is_valid_unicode(value))
     throw std::invalid_argument("Invalid Unicode character: "s + std::to_string(static_cast<uint32_t>(value)));

   } else { // 1 byte: 7 bit ASCII
    value = byte0;
    sequence_length = 1;
   }
  }

  // specialization for UTF-16
  template<>
  void calculate_value<char16_t>()
  {
   size_t remaining{remaining_code_units()};
   
   if (!remaining)
    return;

   char16_t unit0 {get_code_unit<0>()};

   if (unit0 <= 0xD7FF || unit0 >= 0xE000) { // 1 unit (BMP Basic Multilingual Plane)
    value = unit0;
    sequence_length = 1;
   } else {
    if (remaining < 2)
     throw std::invalid_argument("Bad input: Continuation of first UTF-16 unit missing");

    char16_t unit1 {get_code_unit<1>()};
    if ((unit0 & 0xFC00) != 0xD800 || (unit1 & 0xFC00) != 0xDC00)
     throw std::invalid_argument("Bad input: 2 malformed UTF-16 surrogates");

    value = (static_cast<char32_t>(unit0 & 0x03FF) << 10 | (unit1 & 0x03FF)) + 0x10000;
    sequence_length = 2;
   }
  }

  // pre-increment
  utf_iterator<T>& operator++()
  {
   iterator += sequence_length;
   calculate_value<T>();
   return *this;
  }

  bool operator!=(const utf_iterator<T>& other) const
  {
   return iterator != other.iterator;
  }

  reference operator*()
  {
   return value;
  }

 private:
  typename string_type::const_iterator iterator;
  typename string_type::const_iterator end_iterator;

  char32_t value{}; // always save complete unicode code point at this point
  size_t sequence_length{};
 };

 template<typename T>
 struct utf_back_insert_iterator
 {
  typedef T value_type;
  typedef std::basic_string<T> string_type;
  typedef utf_back_insert_iterator& reference;
  typedef utf_back_insert_iterator* pointer;
  typedef size_t difference_type;
  typedef std::output_iterator_tag iterator_category;

  utf_back_insert_iterator(string_type& s): s(s) {}

  utf_back_insert_iterator<T>& operator=(const utf_back_insert_iterator<T>& other)
  {
   if (std::addressof(other.s) != std::addressof(s))
    throw std::runtime_error("utf_back_insert_iterator assignment operator actually called! Iterator should not be assigned to.");

   return *this;
  }

  // no-op
  reference operator++()
  {
   return *this;
  }

  // support *x = value, together with operator=()
  reference operator*()
  {
   return *this;
  }

  // default: utf-32 code unit for UTF-32
  // specializations for UTF-8 and UTF-16 below
  template<typename T1=T>
  reference operator=(const char32_t& value)
  {
   // expect value to be already valid Unicode values
   s.push_back(value);
   return *this;
  }

  // n is number of UTF-8 bytes in sequence
  template<size_t n>
  inline static T byte0_of(char32_t value)
  {
   return (value >> 6 * (n - 1)) | (0xFF << (8 - n));
  }

  // n is index of 6-bit groups, counting from bit 0
  template<size_t n>
  inline static T trailing_byte(char32_t value)
  {
   return ((value >> n * 6) & 0b111111) | 0b10000000;
  }

  // calculate UTF-8 sequence byte for m >= 2 bytes sequences (i.e. non-ASCII)
  // assume value to be valid Unicode value for given byte position
  template<size_t n, size_t m>
  inline static T byte_n_of_m(char32_t value)
  {
   if constexpr (n == 0)
    return byte0_of<m>(value);
   else
    return trailing_byte<m - n - 1>(value);
  }

  // specialization for UTF-8
  // append utf-8 byte sequence
  template<>
  reference operator=<utf8_t>(const char32_t& value)
  {
   if (value < 0x80) { // 1 byte
    s.push_back(static_cast<value_type>(value));
   } else if (value < 0x800) { // 2 bytes
    s.push_back(byte_n_of_m<0,2>(value));
    s.push_back(byte_n_of_m<1,2>(value));
   } else if (value < 0x10000) { // 3 bytes
    s.push_back(byte_n_of_m<0,3>(value));
    s.push_back(byte_n_of_m<1,3>(value));
    s.push_back(byte_n_of_m<2,3>(value));
   } else if (value < 0x110000) { // 4 bytes
    s.push_back(byte_n_of_m<0,4>(value));
    s.push_back(byte_n_of_m<1,4>(value));
    s.push_back(byte_n_of_m<2,4>(value));
    s.push_back(byte_n_of_m<3,4>(value));
   } else
    throw std::runtime_error("Invalid internal Unicode value: "s + std::to_string(static_cast<uint32_t>(value)));
   return *this;
  }

  // specialization for UTF-16
  // append utf-16 word sequence
  template<>
  reference operator=<char16_t>(const char32_t& value)
  {
   if (value <= 0xFFFF) { // expect value to be already valid Unicode values
    s.push_back(static_cast<value_type>(value));
   } else {
    char32_t value_reduced{value - 0x10000};
    s.push_back((value_reduced >> 10) + 0xD800);
    s.push_back((value_reduced & 0x3FF) + 0xDC00);
   }
   return *this;
  }

 private:
  typename utf_back_insert_iterator::string_type& s;
 };

 typedef std::unordered_map<utf8_t, char32_t> iso_map_type;
 typedef std::unordered_map<char32_t, utf8_t> iso_map_type_reverse;

 // ISO-8859-1 is lower 8-bit of Unicode, so no exceptions necessary
 iso_map_type iso_8859_1_map;

 // ISO-8859-15 is lower 8-bit of Unicode, except for:
 iso_map_type iso_8859_15_map {
  { '\xA4', U'\u20AC' }, // €
  { '\xA6', U'\u0160' }, // Š
  { '\xA8', U'\u0161' }, // š
  { '\xB4', U'\u017D' }, // Ž
  { '\xB8', U'\u017E' }, // ž
  { '\xBC', U'\u0152' }, // Œ
  { '\xBD', U'\u0153' }, // œ
  { '\xBE', U'\u0178' }, // Ÿ
 };

 iso_map_type_reverse reverse_iso_map(const iso_map_type& map) {
  iso_map_type_reverse result;
  std::for_each(map.cbegin(), map.cend(),
                [&](const iso_map_type::value_type& pair)
                 {
                  result.emplace(pair.second, pair.first);
                 });
  return result;
 }

 iso_map_type_reverse iso_8859_15_map_reverse { reverse_iso_map(iso_8859_15_map) };
 iso_map_type_reverse iso_8859_1_map_reverse { reverse_iso_map(iso_8859_1_map) };

} // namespace unicode::detail

namespace unicode {

 using namespace detail;

 template<unicode::detail::iso_map_type& Map=iso_8859_1_map>
 struct iso_iterator {
  typedef utf8_t input_type;
  typedef char32_t value_type;
  typedef char32_t& reference;
  typedef char32_t* pointer;
  typedef size_t difference_type;
  typedef std::input_iterator_tag iterator_category;
  typedef std::basic_string<utf8_t>::const_iterator iterator;

  iso_iterator(const iterator& it): m_it(it) {}

  // pre-increment
  iso_iterator& operator++()
  {
   ++m_it;
   return *this;
  }

  bool operator!=(const iso_iterator& other) const
  {
   return m_it != other.m_it;
  }

  // return reference?
  value_type operator*()
  {
   utf8_t value{*m_it};

   if constexpr(std::addressof(Map) != std::addressof(iso_8859_1_map)) // mapping of 128 <= x <= 255 needed
   {
    auto it{Map.find(value)};
    if (it != Map.end())
     return it->second;
   }
   return static_cast<value_type>(static_cast<uint8_t>(value));
  }

 private:
  iterator m_it;
 };

 template<unicode::detail::iso_map_type_reverse& Map=iso_8859_1_map_reverse>
 struct iso_back_insert_iterator {
  typedef iso_back_insert_iterator& reference;
  typedef iso_back_insert_iterator* pointer;
  typedef size_t difference_type;
  typedef utf8_t value_type;
  typedef std::output_iterator_tag iterator_category;
  typedef std::basic_string<utf8_t> string_type;
  
  iso_back_insert_iterator(string_type& s): s(s) {}

  iso_back_insert_iterator& operator=(const iso_back_insert_iterator& other)
  {
   if (std::addressof(other.s) != std::addressof(s))
    throw std::runtime_error("iso_back_insert_iterator assignment operator actually called! Iterator should not be assigned to.");

   return *this;
  }

  // no-op
  reference operator++()
  {
   return *this;
  }

  // support *x = value, together with operator=()
  reference operator*()
  {
   return *this;
  }

  reference operator=(const char32_t& value)
  {
   if constexpr(std::addressof(Map) != std::addressof(iso_8859_1_map_reverse)) // mapping of 128 <= x <= 255 needed
   {
    auto it{Map.find(value)};
    if (it != Map.end()) {
     s.push_back(it->second);
     return *this;
    }
   }

   if (value > 255)
    throw std::invalid_argument("Bad Unicode value above 255: "s + std::to_string(static_cast<uint32_t>(value)));

   s.push_back(static_cast<utf8_t>(value));
   return *this;
  }

 private:
  typename iso_back_insert_iterator::string_type& s;
 };

 // Facet for convert() and ISO-8859-*
 template<typename InputIt, typename OutputIt>
 struct ISO_8859
 {
  typedef utf8_t value_type;

  static InputIt begin(const std::basic_string<value_type>& s)
  {
   return InputIt(s.cbegin());
  }

  static InputIt end(const std::basic_string<value_type>& s)
  {
   return InputIt(s.cend());
  }

  static OutputIt back_inserter(std::basic_string<value_type>& s)
  {
   return OutputIt(s);
  }
 };

 // Facet for convert() and UTF-*
 template<typename InputIt, typename OutputIt>
 struct UTF
 {
  typedef typename InputIt::input_type input_type;
  typedef typename OutputIt::value_type value_type;

  static InputIt begin(const std::basic_string<input_type>& s)
  {
   return InputIt{s.cbegin(), s.cend()};
  }

  static InputIt end(const std::basic_string<input_type>& s)
  {
   return InputIt{s.cend(), s.cend()};
  }

  static OutputIt back_inserter(std::basic_string<value_type>& s)
  {
   return OutputIt(s);
  }
 };

 // Facet for convert()
 typedef ISO_8859<iso_iterator<>, iso_back_insert_iterator<>> ISO_8859_1;
 typedef ISO_8859<iso_iterator<iso_8859_15_map>, iso_back_insert_iterator<iso_8859_15_map_reverse>> ISO_8859_15;
 
 typedef UTF<utf_iterator<utf8_t>, utf_back_insert_iterator<utf8_t>> UTF_8;
 typedef UTF<utf_iterator<char16_t>, utf_back_insert_iterator<char16_t>> UTF_16;
 typedef UTF<utf_iterator<char32_t>, utf_back_insert_iterator<char32_t>> UTF_32;

 // From and To are facets
 template<typename From, typename To>
 std::basic_string<typename To::value_type> convert(const std::basic_string<typename From::value_type>& s)
 {
  std::basic_string<typename To::value_type> result;

  std::copy(From::begin(s), From::end(s), To::back_inserter(result));

  return result;
 }

 // Helper to get correct Facet from char type, e.g. Encoding<typename decltype(s)::value_type>::Facet
 template<typename T>
 struct Encoding
 {
 };

 template<>
 struct Encoding<utf8_t>
 {
  typedef UTF_8 Facet;
 };

 template<>
 struct Encoding<char16_t>
 {
  typedef UTF_16 Facet;
 };

 template<>
 struct Encoding<char32_t>
 {
  typedef UTF_32 Facet;
 };

 // From and To are from: utf8_t, char16_t and char32_t
 template<typename From, typename To>
 std::basic_string<To> convert(const std::basic_string<From>& s)
 {
  std::basic_string<To> result;

  std::copy(Encoding<From>::Facet::begin(s), Encoding<From>::Facet::end(s), Encoding<To>::Facet::back_inserter(result));

  return result;
 }

 template<typename T>
 bool is_valid_utf(const std::basic_string<T>& s)
 {
  try {
   std::for_each(Encoding<T>::Facet::begin(s), Encoding<T>::Facet::end(s), [](const char32_t& c){});
  } catch (const std::invalid_argument&) {
   return false;
  }
  return true;
 }

} // namespace unicode