穿梭於 UTF-8 與 UTF-16 之間－尋找最初的初衷

偶然於 7zip 的 LZMA sdk 裡發現非常簡潔的 UTF-8/UTF-16 變換函數，連一般轉換成 Unicode 的中介動作也省去了。可惜它本身的解壓功能未能滿足遊戲裝載系統的要求，皆因 7zip 的 archive 格式不能以最少的資源去解壓 archive 裡的個別檔案。
以下源始碼引用 LZMA sdk 再加上本人所寫的額外錯誤偵測與註解，enjoy!

轉載自http://mtlung.blogspot.tw/2008/11/utf-8-utf-16.html

typedef byte_t unsigned char;

// Reference: http://en.wikipedia.org/wiki/Utf8

static const byte_t cUtf8Limits[] = {

0xC0, // Start of a 2-byte sequence

0xE0, // Start of a 3-byte sequence

0xF0, // Start of a 4-byte sequence

0xF8, // Start of a 5-byte sequence

0xFC, // Start of a 6-byte sequence

0xFE // Invalid: not defined by original UTF-8 specification

};


/*! Usually it is a 2 steps process to convert the string, invoke utf8ToUtf16() with
dest equals to null so that it gives you destLen (not including null terminator),
then allocate the destination with that amount of memory and call utf8ToUtf16() once
again to perform the actual conversion. You can skip the first call if you sure
the destination buffer is large enough to store the data.
\note Here we assum sizeof(wchar_t) == 2
\ref Modify from 7zip LZMA sdk
*/

bool utf8ToUtf16(wchar_t* dest, size_t& destLen, const char* src, size_t maxSrcLen)
{
size_t destPos = 0, srcPos = 0;
while(true)
{
byte_t c; // Note that byte_t should be unsigned
size_t numAdds;
if(srcPos == maxSrcLen || src[srcPos] == '\0') {
if(dest && destLen != destPos) {
assert(false && "The provided destLen should equals to what we calculated here");
return false;
}
destLen = destPos;
return true;
}
c = src[srcPos++];
if(c < 0x80) { // 0-127, US-ASCII (single byte)
if(dest)
dest[destPos] = (wchar_t)c;
++destPos;
continue;
}
if(c < 0xC0) // The first octet for each code point should within 0-191
break;
for(numAdds = 1; numAdds < 5; ++numAdds)
if(c < cUtf8Limits[numAdds])
break;
uint32_t value = c - cUtf8Limits[numAdds - 1];
do {
byte_t c2;
if(srcPos == maxSrcLen || src[srcPos] == '\0')
break;
c2 = src[srcPos++];
if(c2 < 0x80 || c2 >= 0xC0)
break;
value <<= 6;
value |= (c2 - 0x80);
} while(--numAdds != 0);
if(value < 0x10000) {
if(dest)
dest[destPos] = (wchar_t)value;
++destPos;
}
else {
value -= 0x10000;
if(value >= 0x100000)
break;
if(dest) {
dest[destPos + 0] = (wchar_t)(0xD800 + (value >> 10));
dest[destPos + 1] = (wchar_t)(0xDC00 + (value & 0x3FF));
}
destPos += 2;
}
}
destLen = destPos;
return false;
}

bool utf8ToWStr(const char* utf8Str, size_t maxCount, std::wstring& wideStr)
{
size_t destLen = 0;
// Get the length of the wide string
if(!utf8ToUtf16(nullptr, destLen, utf8Str, maxCount))
return false;
wideStr.resize(destLen);
if(wideStr.size() != destLen)
return false;
return utf8ToUtf16(const_cast<wchar_t*>(wideStr.c_str()), destLen, utf8Str, maxCount);
}

bool utf8ToWStr(const std::string& utf8Str, std::wstring& wideStr)
{
return utf8ToWStr(utf8Str.c_str(), utf8Str.size(), wideStr);
}

//! See the documentation for utf8ToUtf16()
bool utf16ToUtf8(char* dest, size_t& destLen, const wchar_t* src, size_t maxSrcLen)
{
size_t destPos = 0, srcPos = 0;
while(true)
{
uint32_t value;
size_t numAdds;
if(srcPos == maxSrcLen || src[srcPos] == L'\0') {
if(dest && destLen != destPos) {
assert(false && "The provided destLen should equals to what we calculated here");
return false;
}
destLen = destPos;
return true;
}
value = src[srcPos++];
if(value < 0x80) { // 0-127, US-ASCII (single byte)
if(dest)
dest[destPos] = char(value);
++destPos;
continue;
}
if(value >= 0xD800 && value < 0xE000) {
if(value >= 0xDC00 || srcPos == maxSrcLen)
break;
uint32_t c2 = src[srcPos++];
if(c2 < 0xDC00 || c2 >= 0xE000)
break;
value = ((value - 0xD800) << 10) | (c2 - 0xDC00);
}
for(numAdds = 1; numAdds < 5; ++numAdds)
if(value < (uint32_t(1) << (numAdds * 5 + 6)))
break;
if(dest)
dest[destPos] = char(cUtf8Limits[numAdds - 1] + (value >> (6 * numAdds)));
++destPos;
do {
--numAdds;
if(dest)
dest[destPos] = char(0x80 + ((value >> (6 * numAdds)) & 0x3F));
++destPos;
} while(numAdds != 0);
}
destLen = destPos;
return false;
}

bool wStrToUtf8(const wchar_t* wideStr, size_t maxCount, std::string& utf8Str)
{
size_t destLen = 0;
// Get the length of the utf-8 string
if(!utf16ToUtf8(nullptr, destLen, wideStr, maxCount))
return false;
utf8Str.resize(destLen);
if(utf8Str.size() != destLen)
return false;
return utf16ToUtf8(const_cast<char*>(utf8Str.c_str()), destLen, wideStr, maxCount);
}

bool wStrToUtf8(const std::wstring& wideStr, std::string& utf8Str)
{
return wStrToUtf8(wideStr.c_str(), wideStr.size(), utf8Str);
}