charset.cpp

Go to the documentation of this file.

/*
* Character Set Handling
* (C) 1999-2007,2021 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/
 
#include <botan/internal/charset.h>
 
#include <botan/exceptn.h>
#include <botan/internal/loadstor.h>
#include <sstream>
 
namespace Botan {
 
namespace {
 
void append_utf8_for(std::string& s, uint32_t c) {
   if(c >= 0xD800 && c < 0xE000) {
      throw Decoding_Error("Invalid Unicode character");
   }
 
   if(c <= 0x7F) {
      const uint8_t b0 = static_cast<uint8_t>(c);
      s.push_back(static_cast<char>(b0));
   } else if(c <= 0x7FF) {
      const uint8_t b0 = 0xC0 | static_cast<uint8_t>(c >> 6);
      const uint8_t b1 = 0x80 | static_cast<uint8_t>(c & 0x3F);
      s.push_back(static_cast<char>(b0));
      s.push_back(static_cast<char>(b1));
   } else if(c <= 0xFFFF) {
      const uint8_t b0 = 0xE0 | static_cast<uint8_t>(c >> 12);
      const uint8_t b1 = 0x80 | static_cast<uint8_t>((c >> 6) & 0x3F);
      const uint8_t b2 = 0x80 | static_cast<uint8_t>(c & 0x3F);
      s.push_back(static_cast<char>(b0));
      s.push_back(static_cast<char>(b1));
      s.push_back(static_cast<char>(b2));
   } else if(c <= 0x10FFFF) {
      const uint8_t b0 = 0xF0 | static_cast<uint8_t>(c >> 18);
      const uint8_t b1 = 0x80 | static_cast<uint8_t>((c >> 12) & 0x3F);
      const uint8_t b2 = 0x80 | static_cast<uint8_t>((c >> 6) & 0x3F);
      const uint8_t b3 = 0x80 | static_cast<uint8_t>(c & 0x3F);
      s.push_back(static_cast<char>(b0));
      s.push_back(static_cast<char>(b1));
      s.push_back(static_cast<char>(b2));
      s.push_back(static_cast<char>(b3));
   } else {
      throw Decoding_Error("Invalid Unicode character");
   }
}
 
uint32_t next_utf8_codepoint(const std::string& utf8, size_t& pos) {
   auto read_continuation = [&]() -> uint32_t {
      if(pos >= utf8.size()) {
         throw Decoding_Error("Invalid UTF-8 sequence");
      }
      const uint8_t b = static_cast<uint8_t>(utf8[pos++]);
      if((b & 0xC0) != 0x80) {
         throw Decoding_Error("Invalid UTF-8 sequence");
      }
      return b & 0x3F;
   };
 
   const uint8_t lead = static_cast<uint8_t>(utf8[pos++]);
   uint32_t c = 0;
 
   if(lead <= 0x7F) {
      c = lead;
   } else if((lead & 0xE0) == 0xC0) {
      c = (lead & 0x1F) << 6;
      c |= read_continuation();
      if(c < 0x80) {
         throw Decoding_Error("Overlong UTF-8 sequence");
      }
   } else if((lead & 0xF0) == 0xE0) {
      c = (lead & 0x0F) << 12;
      c |= read_continuation() << 6;
      c |= read_continuation();
      if(c < 0x800) {
         throw Decoding_Error("Overlong UTF-8 sequence");
      }
   } else if((lead & 0xF8) == 0xF0) {
      c = (lead & 0x07) << 18;
      c |= read_continuation() << 12;
      c |= read_continuation() << 6;
      c |= read_continuation();
      if(c < 0x10000) {
         throw Decoding_Error("Overlong UTF-8 sequence");
      }
   } else {
      throw Decoding_Error("Invalid UTF-8 sequence");
   }
 
   if(c > 0x10FFFF) {
      throw Decoding_Error("UTF-8 sequence encodes value outside Unicode range");
   }
   if(c >= 0xD800 && c < 0xE000) {
      throw Decoding_Error("UTF-8 sequence encodes surrogate code point");
   }
 
   return c;
}
 
}  // namespace
 
bool is_valid_utf8(const std::string& utf8) {
   try {
      size_t pos = 0;
      while(pos < utf8.size()) {
         const uint32_t c = next_utf8_codepoint(utf8, pos);
         BOTAN_UNUSED(c);
      }
   } catch(Decoding_Error&) {
      return false;
   }
   return true;
}
 
std::string ucs2_to_utf8(const uint8_t ucs2[], size_t len) {
   if(len % 2 != 0) {
      throw Decoding_Error("Invalid length for UCS-2 string");
   }
 
   const size_t chars = len / 2;
 
   std::string s;
   for(size_t i = 0; i != chars; ++i) {
      const uint32_t c = load_be<uint16_t>(ucs2, i);
      append_utf8_for(s, c);
   }
 
   return s;
}
 
std::vector<uint8_t> utf8_to_ucs2(const std::string& utf8) {
   std::vector<uint8_t> out;
   out.reserve(utf8.size() * 2);
 
   size_t pos = 0;
   while(pos < utf8.size()) {
      const uint32_t c = next_utf8_codepoint(utf8, pos);
      if(c > 0xFFFF) {
         throw Decoding_Error("Cannot encode character in UCS-2");
      }
      const uint16_t val = static_cast<uint16_t>(c);
      out.push_back(get_byte<0>(val));
      out.push_back(get_byte<1>(val));
   }
 
   return out;
}
 
std::string ucs4_to_utf8(const uint8_t ucs4[], size_t len) {
   if(len % 4 != 0) {
      throw Decoding_Error("Invalid length for UCS-4 string");
   }
 
   const size_t chars = len / 4;
 
   std::string s;
   for(size_t i = 0; i != chars; ++i) {
      const uint32_t c = load_be<uint32_t>(ucs4, i);
      append_utf8_for(s, c);
   }
 
   return s;
}
 
std::vector<uint8_t> utf8_to_ucs4(const std::string& utf8) {
   std::vector<uint8_t> out;
   out.reserve(utf8.size() * 4);
 
   size_t pos = 0;
   while(pos < utf8.size()) {
      const uint32_t val = next_utf8_codepoint(utf8, pos);
      out.push_back(get_byte<0>(val));
      out.push_back(get_byte<1>(val));
      out.push_back(get_byte<2>(val));
      out.push_back(get_byte<3>(val));
   }
 
   return out;
}
 
/*
* Convert from ISO 8859-1 to UTF-8
*/
std::string latin1_to_utf8(const uint8_t chars[], size_t len) {
   std::string s;
   for(size_t i = 0; i != len; ++i) {
      const uint32_t c = static_cast<uint8_t>(chars[i]);
      append_utf8_for(s, c);
   }
   return s;
}
 
std::string format_char_for_display(char c) {
   std::ostringstream oss;
 
   oss << "'";
 
   if(c == '\t') {
      oss << "\\t";
   } else if(c == '\n') {
      oss << "\\n";
   } else if(c == '\r') {
      oss << "\\r";
   } else if(static_cast<unsigned char>(c) >= 128) {
      const unsigned char z = static_cast<unsigned char>(c);
      oss << "\\x" << std::hex << std::uppercase << static_cast<int>(z);
   } else {
      oss << c;
   }
 
   oss << "'";
 
   return oss.str();
}
 
}  // namespace Botan