字符串编码问题学习
我们native层配置文件用了两套编码，unicode和gbk2312，都是两个字节表示中文。要统一采用gbk编码，顺便顺便熟悉一下gbk。
python3.3的测试
为方便使用python做测试。python默认的字符串类str采用unicode，s = '中文' 等价于 s = u'中文'。
dp_gbk_bs将字符串转化成gbk编码，并输出每个中文对应的unsigned short。
wd_2_gbk_val 将单个字符转化成gbk编码的unsigned short值。
gbk_val_2_wd 将gbk编码下的一个short数值还原成中文字符串。
# function 0
# 输出字符串中每个汉字码值
def dp_gbk_bs(name):
'''
dump gbk2312 string to unsigned short vector.
'''
gbk_bs = str.encode(name, 'gbk')
print( name, "(gbk2312) = ", gbk_bs)
size = len(name)
form = ''
for i in range(size):
form += 'H'
vec = struct.unpack(form, gbk_bs) # return a tuple
# v1, v2 = struct.unpack('HH', gbk_bs)
for i in range(size):
print(name[i], " = ", vec[i])
# function 1
def wd_2_gbk_val(wd):
'''
transform single gbk2312 word to unsigned short value.
中 = 53462 , 0xd0d6
'''
cd_ty = 'gbk'
gbk_bt = str.encode(wd, cd_ty);
vec = struct.unpack('H', gbk_bt) # return tuple type.
return vec[0]
ss = u'中国技术交易大厦'
dp_gbk_bs(ss)
# function 2
# short 转成hex string，然后转化成bytes，最后转化成str(gbk2312)
def gbk_val_2_wd(us_v):
'''
transform unsigned short value to gbk2312 single word.
53462 , 0xd0d6 => 中
'''
hex_s = hex(us_v) # 获取16进制串
print('hex str = ', hex_s)
hex_s = hex_s[2:] # skip '0x' header
hex_list = []
while hex_s:
str_tp = hex_s[0:2]
hex_list.append(str_tp)
hex_s = hex_s[2:]
res = ""
# 大小端倒置，逆序遍历
for w in reversed(hex_list):
res += w
print('reversed hex str = ', res)
bys = bytes.fromhex(res)
print('bytes = ', bys)
return bys.decode('gbk')
string = '中国技术交易大厦'
val_l = []
for ss in string:
val = wd_2_gbk_val(ss)
val_l.append( val )
print( ss, " = ", val)
print(val_l)
for val in val_l:
print('------------------')
print(val, " = ", gbk_val_2_wd(val))
输出结果：
中 = 53462 国 = 64185 技 = 48316 术 = 62922 交 = 48061 易 = 55250 大 = 62388 厦 = 50127 [53462, 64185, 48316, 62922, 48061, 55250, 62388, 50127] ------------------ hex str = 0xd0d6 reversed hex str = d6d0 bytes = b'\xd6\xd0' 53462 = 中 ------------------ hex str = 0xfab9 reversed hex str = b9fa bytes = b'\xb9\xfa' 64185 = 国 ------------------ hex str = 0xbcbc reversed hex str = bcbc bytes = b'\xbc\xbc' 48316 = 技 ------------------ hex str = 0xf5ca reversed hex str = caf5 bytes = b'\xca\xf5' 62922 = 术 ------------------ hex str = 0xbbbd reversed hex str = bdbb bytes = b'\xbd\xbb' 48061 = 交 ------------------ hex str = 0xd7d2 reversed hex str = d2d7 bytes = b'\xd2\xd7' 55250 = 易 ------------------ hex str = 0xf3b4 reversed hex str = b4f3 bytes = b'\xb4\xf3' 62388 = 大 ------------------ hex str = 0xc3cf reversed hex str = cfc3 bytes = b'\xcf\xc3' 50127 = 厦
C的测试
char buf[2] = {0};
short word = 53462; // ‘中’ // 0xd0d6
memcpy(buf, &word, 2); // [-42, -48], 0xd0是-48的补码, 0xd6是-42的补码
// 0xd6 0xd0
unsigned char* byte_ptr = (unsigned char*)&word; // 小端存储
unsigned char byte0 = byte_ptr[0]; // 214 0xd6，0xd6是214的原码
unsigned char byte1 = byte_ptr[1]; // 208 0xd0，0xd0是208的原码
unsigned short word存储汉字‘中’的gbk2312码值，其内容拷贝到char buf数组中，buf内容为负数，buf[0]为word的低字节，buf[1]为高字节。通过byte_ptr指针取得内容为正数。
0xd0d6，word变量内存结构：
内存高地址 | 内存 低地址
d0 | d6
-48 | -42 // signed char，对应的负数补码
208 | 214 // unsigned char 对应的正数原码
buf数组内存结构
buf[0] | buf[1]
取低地址， 取高地址
-42 | -48
Unicode2GBK函数：
/**
* 将unicode字符串转化成gbk2312编码的字符串
*/
int Unicode2GBK( wchar_t *pUnicode, char** ppDest)
{
#ifndef CODE_PAGE_GB18030
#define CODE_PAGE_GB18030 54936
#endif
// get the size of the dest string
const int size = ::WideCharToMultiByte( CODE_PAGE_GB18030, 0/* you can do more for it*/,
pUnicode, -1, 0, 0, 0, 0 );
if ( size == 0 )
{
return -1;
}
char* pDestString = new char[size + 2];
::memset( pDestString, 0, sizeof(pDestString) );
// transform
int ret = ::WideCharToMultiByte( CODE_PAGE_GB18030, 0, pUnicode, -1, pDestString, size, 0, 0 );
if( ret == 0 )
{
delete pDestString; // 失败
return -1;
}
else
{
*ppDest = pDestString;
return 0;
}
return -1;
}
refer：windows下 unicode转化成gbk： http://hi.baidu.com/zhangweijiqn/item/e2ca4c1acfcb42d4bf904284