39 static void SkipUtf8Bom(FILE* fp);
47 if ((ch & 0xF0) == 0xE0) {
49 }
else if ((ch & 0x80) == 0x00) {
51 }
else if ((ch & 0xE0) == 0xC0) {
53 }
else if ((ch & 0xF8) == 0xF0) {
55 }
else if ((ch & 0xFC) == 0xF8) {
57 }
else if ((ch & 0xFE) == 0xFC) {
67 size_t length = NextCharLengthNoException(str);
79 const size_t length = NextCharLengthNoException(str - 3);
85 const size_t length = NextCharLengthNoException(str - 1);
91 const size_t length = NextCharLengthNoException(str - 2);
96 for (
size_t i = 4; i <= 6; i++) {
97 const size_t length = NextCharLengthNoException(str - i);
109 return str + NextCharLength(str);
116 return str - PrevCharLength(str);
124 while (*str !=
'\0') {
138 while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
148 return ch ==
'\0' || ch ==
'\n' || ch ==
'\r';
156 newStr.resize(length);
157 strncpy(
const_cast<char*
>(newStr.c_str()), str, length);
165 while (byteLength > 0) {
181 if (NotShorterThan(str, maxByteLength)) {
183 const char* pStr = str;
185 const size_t charLength = NextCharLength(pStr);
186 if (len + charLength > maxByteLength) {
192 wordTrunc = FromSubstr(str, len);
202 static void ReplaceAll(
string& str,
const char* from,
const char* to) {
203 string::size_type pos = 0;
204 string::size_type fromLen = strlen(from);
205 string::size_type toLen = strlen(to);
206 while ((pos = str.find(from, pos)) != string::npos) {
207 str.replace(pos, fromLen, to);
215 static string Join(
const vector<string>& strings,
const string& separator) {
216 std::ostringstream buffer;
218 for (
const auto& str : strings) {
231 static string Join(
const vector<string>& strings) {
232 std::ostringstream buffer;
233 for (
const auto& str : strings) {
239 static void GetByteMap(
const char* str,
const size_t utf8Length,
240 vector<size_t>* byteMap) {
241 if (byteMap->size() < utf8Length) {
242 byteMap->resize(utf8Length);
244 const char* pstr = str;
245 for (
size_t i = 0; i < utf8Length; i++) {
246 (*byteMap)[i] = pstr - str;
247 pstr = NextChar(pstr);
252 static std::wstring GetPlatformString(
const std::string& str) {
256 static std::string GetPlatformString(
const std::string& str) {
263 static std::string U16ToU8(
const std::wstring& wstr) {
265 int length =
static_cast<int>(wstr.length());
266 int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0, NULL, NULL);
269 WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt, NULL, NULL);
274 static std::wstring U8ToU16(
const std::string& str) {
276 int length =
static_cast<int>(str.length());
277 int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
280 MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);