#ifndef _coding_conv_ns_h__ #define _coding_conv_ns_h__ #include #include #include #include #if defined(_MSC_VER) || defined(WIN64) || defined(WIN32) #include #else #include #endif namespace coding_conv_ns { static int preNUm(unsigned char byted) { unsigned char mask = 0x80; int num = 0; for (int i = 0; i < 8; i++) { if ((byted & mask) == mask) { mask = mask >> 1; num++; } else { break; } } return num; } static bool isUtf8(unsigned char* data, int len) { int num = 0; int i = 0; while (i < len) { if ((data[i] & 0x80) == 0x00) { // 0XXX_XXXX i++; continue; } else if ((num = preNUm(data[i])) > 2) { // 110X_XXXX 10XX_XXXX // 1110_XXXX 10XX_XXXX 10XX_XXXX // 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX // 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX // 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX // preNUm() 返回首个字节8个bits中首??0bit前面1bit的个数,该数量也是该字符所使用的字节数 i++; for(int j = 0; j < num - 1; j++) { //判断后面num - 1 个字节是不是都是10开 if ((data[i] & 0xc0) != 0x80) { return false; } i++; } } else { //其他情况说明不是utf-8 return false; } } return true; } static bool isGBK(unsigned char* data, int len) { int i = 0; while (i < len) { if (data[i] <= 0x7f) { //编码小于等于127,只有一个字节的编码,兼容ASCII i++; continue; } else { //大于127的使用双字节编码 if (data[i] >= 0x81 && data[i] <= 0xfe && data[i + 1] >= 0x40 && data[i + 1] <= 0xfe && data[i + 1] != 0xf7) { i += 2; continue; } else { return false; } } } return true; } typedef enum { GBK, UTF8, UNKOWN } CODING; //需要说明的是,isGBK()是通过双字节是否落在gbk的编码范围内实现的, //而utf-8编码格式的每个字节都是落在gbk的编码范围内?? //所以只有先调用isUtf8()先判断不是utf-8编码,再调用isGBK()才有意义 static CODING GetCoding(const char* data, int len) { CODING coding; if (isUtf8((unsigned char*)(data), len) == true) { coding = UTF8; } else if (isGBK((unsigned char*)(data), len) == true) { coding = GBK; } else { coding = UNKOWN; } return coding; } #if defined(_MSC_VER) || defined(WIN64) || defined(WIN32) static std::string utf8_to_gbk_str(const char* src) { auto code = GetCoding(src, strlen(src)); if (code != UTF8) { return src; } std::string gbkstr; wchar_t* pUnicodeBuff = NULL; int rlen = 0; rlen = MultiByteToWideChar(CP_UTF8, 0, src, -1, NULL, NULL); pUnicodeBuff = new wchar_t[rlen + 1]; //为Unicode字符串空间 rlen = MultiByteToWideChar(CP_UTF8, 0, src, -1, pUnicodeBuff, rlen); rlen = WideCharToMultiByte(936, 0, pUnicodeBuff, -1, NULL, NULL, NULL, NULL); //936 为windows gb2312代码页码 char* dstgbk = new char[rlen + 1]; WideCharToMultiByte(936, 0, pUnicodeBuff, -1, dstgbk, rlen, NULL, NULL); delete[] pUnicodeBuff; gbkstr = dstgbk; delete[]dstgbk; return gbkstr; } static std::string gbk_to_utf8_str(const char* src) { auto code = GetCoding(src, strlen(src)); if (code == UTF8) { return src; } int rlen = MultiByteToWideChar(936, 0, src, -1, NULL, NULL); wchar_t* pUnicodeBuff = new wchar_t[rlen + 1]; //为Unicode字符串空间 rlen = MultiByteToWideChar(936, 0, src, -1, pUnicodeBuff, rlen); rlen = WideCharToMultiByte(CP_UTF8, 0, pUnicodeBuff, -1, NULL, NULL, NULL, NULL); char* dstutf8 = new char[rlen + 1]; WideCharToMultiByte(CP_UTF8, 0, pUnicodeBuff, -1, dstutf8, rlen, NULL, NULL); delete[] pUnicodeBuff; std::string str = dstutf8; delete[]dstutf8; return str; } #else static std::string utf8_to_gbk_str(const char* src) { auto code = GetCoding(src, strlen(src)); if (code == UTF8) { std::string strGBK = boost::locale::conv::from_utf(src, "GBK"); return strGBK; } return src; } static std::string gbk_to_utf8_str(const char* src) { auto code =GetCoding(src, strlen(src)); if (code == UTF8) { return src; } std::string strUTF8 = boost::locale::conv::to_utf(src, "GBK"); return strUTF8; } #endif // WIN32 || WIN64 //在linux 下如果是 gbk则转换为utf8 //在windows 下如果是 utf8则转换为gbk static std::string check_str_to_default(const char* str) { #if defined(_MSC_VER) || defined(WIN64) || defined(WIN32) if (GetCoding(str, strlen(str) == UTF8)) { return utf8_to_gbk_str(str); } #else auto code = GetCoding(str, strlen(str)); if (code != UTF8) { return gbk_to_utf8_str(str); } #endif return str; } } #endif // _coding_conv_ns_h__