myUTF-8小库(验证UTF-8，猜测语言，计数字符)
myUTF-8小库(验证UTF-8，猜测语言，计数字符)
提问于 2019-04-10 01:36:50
我对C语言很陌生，从来不了解UTF-8的细节，在阅读了一些文章关于它的文章之后，我想尝试用C语言玩UTF-8，既好玩又练习。这是一个小小的C库，用于验证UTF-8字符串、计数字符、根据Unicode代码点块猜测字符串的语言。
- 你对此有何看法？
- 这是我第一次设计一个C库，你认为它是一个库吗？
- 另外，您对我组织源文件/保护代码/公开代码(软件设计)的方式有什么看法？
非常感谢
源代码由3个文件组成。
libmyutf8.c
这是库的全部代码。
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include "headerAll.h"
unsigned long buildCodePoint(unsigned char byte1,
unsigned char byte2,
unsigned char byte3,
unsigned char byte4,
int numberOfBytes );
int increseCodeBlock( unsigned long codePoint,
int codeBlocksCount,
struct codePointBlock *codePointBlocks);
//macros
//https://stackoverflow.com/questions/523724/c-c-check-if-one-bit-is-set-in-i-e-int-variable
#define CHECK_BIT(var,pos) ((var) & (1<<(pos)) )
unsigned long scanUTF8(char *str, struct codePointBlock *codePointBlocks)
{
int i = 0;
unsigned long maxLength = 10485760; // 10MB
unsigned char currentByte;
unsigned char byte1 = 0, byte2 = 0, byte3 = 0, byte4 = 0;
int firstByte = 1;
int charBytesRemaining = 0;
unsigned long codePoint = 0;
int numberOfBytes = 0;
int codeBlocksCount = 0;
int charsFound = 0;
while(codePointBlocks[codeBlocksCount].blockName) codeBlocksCount++;
do {
currentByte = *(str + i);
//printf("iiiiiiiiiiiiiii%d\n", i);
//printf("curentByteee%d\n", currentByte);
if(currentByte == 0) break;//end of string
if (firstByte) {
firstByte = 0;
//lastCodePoint is ready
byte1 = currentByte;
if ( !(CHECK_BIT(currentByte, 7)) ){
// if 0XXXXXXX
firstByte = 1;
numberOfBytes = 1;
charBytesRemaining = 0;
codePoint = buildCodePoint(byte1, 0, 0, 0, numberOfBytes);
increseCodeBlock(codePoint, codeBlocksCount, codePointBlocks);
charsFound++;
} else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
!CHECK_BIT(currentByte, 5)
){
// if 110XXXXX
charBytesRemaining = 1;
numberOfBytes = 2;
} else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
CHECK_BIT(currentByte, 5) && !CHECK_BIT(currentByte, 4)
){
// if 1110XXXX
charBytesRemaining = 2;
numberOfBytes = 3;
} else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
CHECK_BIT(currentByte, 5) && CHECK_BIT(currentByte, 4) &&
!CHECK_BIT(currentByte, 3)
){
// if 11110XXX
charBytesRemaining = 3;
numberOfBytes = 4;
} else {
//not utf-8
return -1;
}
} else {
//not first byte in char
if (CHECK_BIT(currentByte, 7) && !CHECK_BIT(currentByte, 6)){
//must be 10XXXXXX
// byte1 byte2 byte3 byte4
if (charBytesRemaining == 3){
byte2 = currentByte;
} else if(charBytesRemaining == 2){
if (numberOfBytes == 4){
byte3 = currentByte;
} else if (numberOfBytes == 3){
byte2 = currentByte;
}
} else if (charBytesRemaining == 1){
if (numberOfBytes == 4){
byte4 = currentByte;
} else if (numberOfBytes == 3){
byte3 = currentByte;
} else if (numberOfBytes == 2){
byte2 = currentByte;
}
}
charBytesRemaining--;
if (charBytesRemaining == 0){
//end of char (last byte)
firstByte = 1;
/*
// uncomment for debugging
printf("codePointCalled\n");
printf("byte1:%d\n", byte1);
printf("byte2:%d\n", byte2);
printf("byte3:%d\n", byte3);
printf("byte4:%d\n", byte4);
printf("numberOfBytes:%d\n", numberOfBytes);
*/
codePoint = buildCodePoint(byte1, byte2, byte3, byte4, numberOfBytes);
increseCodeBlock(codePoint, codeBlocksCount, codePointBlocks);
byte1 = byte2 = byte3 = byte4 = numberOfBytes = 0;
charsFound++;
}
}else{
//not utf-8
return -1;
}
}
i++;
} while (i < maxLength);
return charsFound;
}
int increseCodeBlock( unsigned long codePoint,
int codeBlocksCount,
struct codePointBlock *codePointBlocks)
{
for(int i = 0; i < codeBlocksCount; i++){
if(codePoint >= codePointBlocks[i].start && codePoint <= codePointBlocks[i].end){
codePointBlocks[i].count++;
return 1;
}
}
return 0;
}
unsigned long buildCodePoint(unsigned char byte1,
unsigned char byte2,
unsigned char byte3,
unsigned char byte4,
int numberOfBytes )
{
// codePoint = (Byte1) | (Byte2) | (Byte3) | (Byte4)
if(numberOfBytes == 1){
byte2 = 0;
byte3 = 0;
byte4 = 0;
}else if(numberOfBytes == 2){
byte1 &= 0b00011111;
byte2 &= 0b00111111;
byte3 = 0;
byte4 = 0;
}else if(numberOfBytes == 3){
byte1 &= 0b00001111;
byte2 &= 0b00111111;
byte3 &= 0b00111111;
byte4 = 0;
}else if(numberOfBytes == 4){
byte1 &= 0b00000111;
byte2 &= 0b00111111;
byte3 &= 0b00111111;
byte4 &= 0b00111111;
}else {
perror("buildCodePoint number of bytes is not riht");
return 0;
}
if(numberOfBytes == 1) return (unsigned long) byte1;
unsigned long byte1L = (unsigned long) byte1;
unsigned long byte2L = (unsigned long) byte2;
unsigned long byte3L = (unsigned long) byte3;
unsigned long byte4L = (unsigned long) byte4;
int missedBits = (4 - numberOfBytes) * 8;// 0 4
// 1(8) 3
// 2(16) 2
// codePoint = (Byte1) | (Byte2) | (Byte3) | (Byte4)
//
unsigned long codePoint = byte4L |
(byte3L << (8 - (missedBits + 2 * (numberOfBytes - 3) ) ) )|
(byte2L << (16 - (missedBits + 2 * (numberOfBytes - 2) )) ) |
(byte1L << (24 - (missedBits + 2 * (numberOfBytes - 1) )));
return codePoint;
}
//code blocks extracted from https://www.utf8-chartable.de/unicode-utf8-table.pl
struct codePointBlock codePointBlocks[] = {
//281 blocks
{0x0000, 0x007F, "Basic Latin",0},
{0x0080, 0x00FF, "Latin-1 Supplement",0},
{0x0100, 0x017F, "Latin Extended-A",0},
{0x0180, 0x024F, "Latin Extended-B",0},
{0x0250, 0x02AF, "IPA Extensions",0},
{0x02B0, 0x02FF, "Spacing Modifier Letters",0},
{0x0300, 0x036F, "Combining Diacritical Marks",0},
{0x0370, 0x03FF, "Greek and Coptic",0},
{0x0400, 0x04FF, "Cyrillic",0},
{0x0500, 0x052F, "Cyrillic Supplement",0},
{0x0530, 0x058F, "Armenian",0},
{0x0590, 0x05FF, "Hebrew",0},
{0x0600, 0x06FF, "Arabic",0},
{0x0700, 0x074F, "Syriac",0},
{0x0750, 0x077F, "Arabic Supplement",0},
{0x0780, 0x07BF, "Thaana",0},
{0x07C0, 0x07FF, "NKo",0},
{0x0800, 0x083F, "Samaritan",0},
{0x0840, 0x085F, "Mandaic",0},
{0x0860, 0x086F, "Syriac Supplement",0},
{0x08A0, 0x08FF, "Arabic Extended-A",0},
{0x0900, 0x097F, "Devanagari",0},
{0x0980, 0x09FF, "Bengali",0},
{0x0A00, 0x0A7F, "Gurmukhi",0},
{0x0A80, 0x0AFF, "Gujarati",0},
{0x0B00, 0x0B7F, "Oriya",0},
{0x0B80, 0x0BFF, "Tamil",0},
{0x0C00, 0x0C7F, "Telugu",0},
{0x0C80, 0x0CFF, "Kannada",0},
{0x0D00, 0x0D7F, "Malayalam",0},
{0x0D80, 0x0DFF, "Sinhala",0},
{0x0E00, 0x0E7F, "Thai",0},
{0x0E80, 0x0EFF, "Lao",0},
{0x0F00, 0x0FFF, "Tibetan",0},
{0x1000, 0x109F, "Myanmar",0},
{0x10A0, 0x10FF, "Georgian",0},
{0x1100, 0x11FF, "Hangul Jamo",0},
{0x1200, 0x137F, "Ethiopic",0},
{0x1380, 0x139F, "Ethiopic Supplement",0},
{0x13A0, 0x13FF, "Cherokee",0},
{0x1400, 0x167F, "Unified Canadian Aboriginal Syllabics",0},
{0x1680, 0x169F, "Ogham",0},
{0x16A0, 0x16FF, "Runic",0},
{0x1700, 0x171F, "Tagalog",0},
{0x1720, 0x173F, "Hanunoo",0},
{0x1740, 0x175F, "Buhid",0},
{0x1760, 0x177F, "Tagbanwa",0},
{0x1780, 0x17FF, "Khmer",0},
{0x1800, 0x18AF, "Mongolian",0},
{0x18B0, 0x18FF, "Unified Canadian Aboriginal Syllabics Extended",0},
{0x1900, 0x194F, "Limbu",0},
{0x1950, 0x197F, "Tai Le",0},
{0x1980, 0x19DF, "New Tai Lue",0},
{0x19E0, 0x19FF, "Khmer Symbols",0},
{0x1A00, 0x1A1F, "Buginese",0},
{0x1A20, 0x1AAF, "Tai Tham",0},
{0x1AB0, 0x1AFF, "Combining Diacritical Marks Extended",0},
{0x1B00, 0x1B7F, "Balinese",0},
{0x1B80, 0x1BBF, "Sundanese",0},
{0x1BC0, 0x1BFF, "Batak",0},
{0x1C00, 0x1C4F, "Lepcha",0},
{0x1C50, 0x1C7F, "Ol Chiki",0},
{0x1C80, 0x1C8F, "Cyrillic Extended-C",0},
{0x1CC0, 0x1CCF, "Sundanese Supplement",0},
{0x1CD0, 0x1CFF, "Vedic Extensions",0},
{0x1D00, 0x1D7F, "Phonetic Extensions",0},
{0x1D80, 0x1DBF, "Phonetic Extensions Supplement",0},
{0x1DC0, 0x1DFF, "Combining Diacritical Marks Supplement",0},
{0x1E00, 0x1EFF, "Latin Extended Additional",0},
{0x1F00, 0x1FFF, "Greek Extended",0},
{0x2000, 0x206F, "General Punctuation",0},
{0x2070, 0x209F, "Superscripts and Subscripts",0},
{0x20A0, 0x20CF, "Currency Symbols",0},
{0x20D0, 0x20FF, "Combining Diacritical Marks for Symbols",0},
{0x2100, 0x214F, "Letterlike Symbols",0},
{0x2150, 0x218F, "Number Forms",0},
{0x2190, 0x21FF, "Arrows",0},
{0x2200, 0x22FF, "Mathematical Operators",0},
{0x2300, 0x23FF, "Miscellaneous Technical",0},
{0x2400, 0x243F, "Control Pictures",0},
{0x2440, 0x245F, "Optical Character Recognition",0},
{0x2460, 0x24FF, "Enclosed Alphanumerics",0},
{0x2500, 0x257F, "Box Drawing",0},
{0x2580, 0x259F, "Block Elements",0},
{0x25A0, 0x25FF, "Geometric Shapes",0},
{0x2600, 0x26FF, "Miscellaneous Symbols",0},
{0x2700, 0x27BF, "Dingbats",0},
{0x27C0, 0x27EF, "Miscellaneous Mathematical Symbols-A",0},
{0x27F0, 0x27FF, "Supplemental Arrows-A",0},
{0x2800, 0x28FF, "Braille Patterns",0},
{0x2900, 0x297F, "Supplemental Arrows-B",0},
{0x2980, 0x29FF, "Miscellaneous Mathematical Symbols-B",0},
{0x2A00, 0x2AFF, "Supplemental Mathematical Operators",0},
{0x2B00, 0x2BFF, "Miscellaneous Symbols and Arrows",0},
{0x2C00, 0x2C5F, "Glagolitic",0},
{0x2C60, 0x2C7F, "Latin Extended-C",0},
{0x2C80, 0x2CFF, "Coptic",0},
{0x2D00, 0x2D2F, "Georgian Supplement",0},
{0x2D30, 0x2D7F, "Tifinagh",0},
{0x2D80, 0x2DDF, "Ethiopic Extended",0},
{0x2DE0, 0x2DFF, "Cyrillic Extended-A",0},
{0x2E00, 0x2E7F, "Supplemental Punctuation",0},
{0x2E80, 0x2EFF, "CJK Radicals Supplement",0},
{0x2F00, 0x2FDF, "Kangxi Radicals",0},
{0x2FF0, 0x2FFF, "Ideographic Description Characters",0},
{0x3000, 0x303F, "CJK Symbols and Punctuation",0},
{0x3040, 0x309F, "Hiragana",0},
{0x30A0, 0x30FF, "Katakana",0},
{0x3100, 0x312F, "Bopomofo",0},
{0x3130, 0x318F, "Hangul Compatibility Jamo",0},
{0x3190, 0x319F, "Kanbun",0},
{0x31A0, 0x31BF, "Bopomofo Extended",0},
{0x31C0, 0x31EF, "CJK Strokes",0},
{0x31F0, 0x31FF, "Katakana Phonetic Extensions",0},
{0x3200, 0x32FF, "Enclosed CJK Letters and Months",0},
{0x3300, 0x33FF, "CJK Compatibility",0},
{0x3400, 0x4DBF, "CJK Unified Ideographs Extension A",0},
{0x4DC0, 0x4DFF, "Yijing Hexagram Symbols",0},
{0x4E00, 0x9FFF, "CJK Unified Ideographs",0},
{0xA000, 0xA48F, "Yi Syllables",0},
{0xA490, 0xA4CF, "Yi Radicals",0},
{0xA4D0, 0xA4FF, "Lisu",0},
{0xA500, 0xA63F, "Vai",0},
{0xA640, 0xA69F, "Cyrillic Extended-B",0},
{0xA6A0, 0xA6FF, "Bamum",0},
{0xA700, 0xA71F, "Modifier Tone Letters",0},
{0xA720, 0xA7FF, "Latin Extended-D",0},
{0xA800, 0xA82F, "Syloti Nagri",0},
{0xA830, 0xA83F, "Common Indic Number Forms",0},
{0xA840, 0xA87F, "Phags-pa",0},
{0xA880, 0xA8DF, "Saurashtra",0},
{0xA8E0, 0xA8FF, "Devanagari Extended",0},
{0xA900, 0xA92F, "Kayah Li",0},
{0xA930, 0xA95F, "Rejang",0},
{0xA960, 0xA97F, "Hangul Jamo Extended-A",0},
{0xA980, 0xA9DF, "Javanese",0},
{0xA9E0, 0xA9FF, "Myanmar Extended-B",0},
{0xAA00, 0xAA5F, "Cham",0},
{0xAA60, 0xAA7F, "Myanmar Extended-A",0},
{0xAA80, 0xAADF, "Tai Viet",0},
{0xAAE0, 0xAAFF, "Meetei Mayek Extensions",0},
{0xAB00, 0xAB2F, "Ethiopic Extended-A",0},
{0xAB30, 0xAB6F, "Latin Extended-E",0},
{0xAB70, 0xABBF, "Cherokee Supplement",0},
{0xABC0, 0xABFF, "Meetei Mayek",0},
{0xAC00, 0xD7AF, "Hangul Syllables",0},
{0xD7B0, 0xD7FF, "Hangul Jamo Extended-B",0},
{0xD800, 0xDB7F, "High Surrogates",0},
{0xDB80, 0xDBFF, "High Private Use Surrogates",0},
{0xDC00, 0xDFFF, "Low Surrogates",0},
{0xE000, 0xF8FF, "Private Use Area",0},
{0xF900, 0xFAFF, "CJK Compatibility Ideographs",0},
{0xFB00, 0xFB4F, "Alphabetic Presentation Forms",0},
{0xFB50, 0xFDFF, "Arabic Presentation Forms-A",0},
{0xFE00, 0xFE0F, "Variation Selectors",0},
{0xFE10, 0xFE1F, "Vertical Forms",0},
{0xFE20, 0xFE2F, "Combining Half Marks",0},
{0xFE30, 0xFE4F, "CJK Compatibility Forms",0},
{0xFE50, 0xFE6F, "Small Form Variants",0},
{0xFE70, 0xFEFF, "Arabic Presentation Forms-B",0},
{0xFF00, 0xFFEF, "Halfwidth and Fullwidth Forms",0},
{0xFFF0, 0xFFFF, "Specials",0},
{0x10000, 0x1007F, "Linear B Syllabary",0},
{0x10080, 0x100FF, "Linear B Ideograms",0},
{0x10100, 0x1013F, "Aegean Numbers",0},
{0x10140, 0x1018F, "Ancient Greek Numbers",0},
{0x10190, 0x101CF, "Ancient Symbols",0},
{0x101D0, 0x101FF, "Phaistos Disc",0},
{0x10280, 0x1029F, "Lycian",0},
{0x102A0, 0x102DF, "Carian",0},
{0x102E0, 0x102FF, "Coptic Epact Numbers",0},
{0x10300, 0x1032F, "Old Italic",0},
{0x10330, 0x1034F, "Gothic",0},
{0x10350, 0x1037F, "Old Permic",0},
{0x10380, 0x1039F, "Ugaritic",0},
{0x103A0, 0x103DF, "Old Persian",0},
{0x10400, 0x1044F, "Deseret",0},
{0x10450, 0x1047F, "Shavian",0},
{0x10480, 0x104AF, "Osmanya",0},
{0x104B0, 0x104FF, "Osage",0},
{0x10500, 0x1052F, "Elbasan",0},
{0x10530, 0x1056F, "Caucasian Albanian",0},
{0x10600, 0x1077F, "Linear A",0},
{0x10800, 0x1083F, "Cypriot Syllabary",0},
{0x10840, 0x1085F, "Imperial Aramaic",0},
{0x10860, 0x1087F, "Palmyrene",0},
{0x10880, 0x108AF, "Nabataean",0},
{0x108E0, 0x108FF, "Hatran",0},
{0x10900, 0x1091F, "Phoenician",0},
{0x10920, 0x1093F, "Lydian",0},
{0x10980, 0x1099F, "Meroitic Hieroglyphs",0},
{0x109A0, 0x109FF, "Meroitic Cursive",0},
{0x10A00, 0x10A5F, "Kharoshthi",0},
{0x10A60, 0x10A7F, "Old South Arabian",0},
{0x10A80, 0x10A9F, "Old North Arabian",0},
{0x10AC0, 0x10AFF, "Manichaean",0},
{0x10B00, 0x10B3F, "Avestan",0},
{0x10B40, 0x10B5F, "Inscriptional Parthian",0},
{0x10B60, 0x10B7F, "Inscriptional Pahlavi",0},
{0x10B80, 0x10BAF, "Psalter Pahlavi",0},
{0x10C00, 0x10C4F, "Old Turkic",0},
{0x10C80, 0x10CFF, "Old Hungarian",0},
{0x10E60, 0x10E7F, "Rumi Numeral Symbols",0},
{0x11000, 0x1107F, "Brahmi",0},
{0x11080, 0x110CF, "Kaithi",0},
{0x110D0, 0x110FF, "Sora Sompeng",0},
{0x11100, 0x1114F, "Chakma",0},
{0x11150, 0x1117F, "Mahajani",0},
{0x11180, 0x111DF, "Sharada",0},
{0x111E0, 0x111FF, "Sinhala Archaic Numbers",0},
{0x11200, 0x1124F, "Khojki",0},
{0x11280, 0x112AF, "Multani",0},
{0x112B0, 0x112FF, "Khudawadi",0},
{0x11300, 0x1137F, "Grantha",0},
{0x11400, 0x1147F, "Newa",0},
{0x11480, 0x114DF, "Tirhuta",0},
{0x11580, 0x115FF, "Siddham",0},
{0x11600, 0x1165F, "Modi",0},
{0x11660, 0x1167F, "Mongolian Supplement",0},
{0x11680, 0x116CF, "Takri",0},
{0x11700, 0x1173F, "Ahom",0},
{0x118A0, 0x118FF, "Warang Citi",0},
{0x11A00, 0x11A4F, "Zanabazar Square",0},
{0x11A50, 0x11AAF, "Soyombo",0},
{0x11AC0, 0x11AFF, "Pau Cin Hau",0},
{0x11C00, 0x11C6F, "Bhaiksuki",0},
{0x11C70, 0x11CBF, "Marchen",0},
{0x11D00, 0x11D5F, "Masaram Gondi",0},
{0x12000, 0x123FF, "Cuneiform",0},
{0x12400, 0x1247F, "Cuneiform Numbers and Punctuation",0},
{0x12480, 0x1254F, "Early Dynastic Cuneiform",0},
{0x13000, 0x1342F, "Egyptian Hieroglyphs",0},
{0x14400, 0x1467F, "Anatolian Hieroglyphs",0},
{0x16800, 0x16A3F, "Bamum Supplement",0},
{0x16A40, 0x16A6F, "Mro",0},
{0x16AD0, 0x16AFF, "Bassa Vah",0},
{0x16B00, 0x16B8F, "Pahawh Hmong",0},
{0x16F00, 0x16F9F, "Miao",0},
{0x16FE0, 0x16FFF, "Ideographic Symbols and Punctuation",0},
{0x17000, 0x187FF, "Tangut",0},
{0x18800, 0x18AFF, "Tangut Components",0},
{0x1B000, 0x1B0FF, "Kana Supplement",0},
{0x1B100, 0x1B12F, "Kana Extended-A",0},
{0x1B170, 0x1B2FF, "Nushu",0},
{0x1BC00, 0x1BC9F, "Duployan",0},
{0x1BCA0, 0x1BCAF, "Shorthand Format Controls",0},
{0x1D000, 0x1D0FF, "Byzantine Musical Symbols",0},
{0x1D100, 0x1D1FF, "Musical Symbols",0},
{0x1D200, 0x1D24F, "Ancient Greek Musical Notation",0},
{0x1D300, 0x1D35F, "Tai Xuan Jing Symbols",0},
{0x1D360, 0x1D37F, "Counting Rod Numerals",0},
{0x1D400, 0x1D7FF, "Mathematical Alphanumeric Symbols",0},
{0x1D800, 0x1DAAF, "Sutton SignWriting",0},
{0x1E000, 0x1E02F, "Glagolitic Supplement",0},
{0x1E800, 0x1E8DF, "Mende Kikakui",0},
{0x1E900, 0x1E95F, "Adlam",0},
{0x1EE00, 0x1EEFF, "Arabic Mathematical Alphabetic Symbols",0},
{0x1F000, 0x1F02F, "Mahjong Tiles",0},
{0x1F030, 0x1F09F, "Domino Tiles",0},
{0x1F0A0, 0x1F0FF, "Playing Cards",0},
{0x1F100, 0x1F1FF, "Enclosed Alphanumeric Supplement",0},
{0x1F200, 0x1F2FF, "Enclosed Ideographic Supplement",0},
{0x1F300, 0x1F5FF, "Miscellaneous Symbols and Pictographs",0},
{0x1F600, 0x1F64F, "Emoticons",0},
{0x1F650, 0x1F67F, "Ornamental Dingbats",0},
{0x1F680, 0x1F6FF, "Transport and Map Symbols",0},
{0x1F700, 0x1F77F, "Alchemical Symbols",0},
{0x1F780, 0x1F7FF, "Geometric Shapes Extended",0},
{0x1F800, 0x1F8FF, "Supplemental Arrows-C",0},
{0x1F900, 0x1F9FF, "Supplemental Symbols and Pictographs",0},
{0x20000, 0x2A6DF, "CJK Unified Ideographs Extension B",0},
{0x2A700, 0x2B73F, "CJK Unified Ideographs Extension C",0},
{0x2B740, 0x2B81F, "CJK Unified Ideographs Extension D",0},
{0x2B820, 0x2CEAF, "CJK Unified Ideographs Extension E",0},
{0x2CEB0, 0x2EBEF, "CJK Unified Ideographs Extension F",0},
{0x2F800, 0x2FA1F, "CJK Compatibility Ideographs Supplement",0},
{0xE0000, 0xE007F, "Tags",0},
{0xE0100, 0xE01EF, "Variation Selectors Supplement",0},
{0xF0000, 0xFFFFF, "Supplementary Private Use Area-A",0},
{0x100000, 0x10FFFF, "Supplementary Private Use Area-B",0},
{0, 0xFFFFFF, "Unknown",0},
{0,0,NULL,0}
};libmyutf8.h
这只是库用户将要包含的内容(库接口)。只有一个函数
#ifndef HEADER_MYLIB
#define HEADER_MYLIB
#include "headerAll.h"
extern unsigned long scanUTF8(char *str, struct codePointBlock *codePointBlocks );
extern struct codePointBlock codePointBlocks;
#endifheaderAll.h
此头文件将包含库文件和用户文件所需的定义。
#ifndef HEADER_ALL
#define HEADER_ALL
struct codePointBlock{
int start;
int end;
char *blockName;
int count;
};
#endif测试
这是用户文件
#include <stdio.h>
#include <stdlib.h>
#include "libmyutf8.h" // < the library
int main( int argc, char *argv[])
{
FILE* pFile;
char *buffer = 0;
pFile = fopen(argv[1], "rb");
if(pFile == NULL) return 1;
fseek (pFile, 0, SEEK_END);
long length = ftell (pFile);
fseek (pFile, 0, SEEK_SET);
buffer = malloc (length + 1);
if (buffer) fread (buffer, 1, length, pFile);
fclose (pFile);
buffer[length] = '\0';
struct codePointBlock *cpbPointer = &codePointBlocks;
int charsCount = scanUTF8(buffer, cpbPointer);
int i = 0;
while(cpbPointer[i].blockName){
if(cpbPointer[i].count) printf("%s: %d\n", cpbPointer[i].blockName, cpbPointer[i].count);
i++;
}
printf("chars count:%d\n", charsCount);
return 0;
}我是这样测试的
./libmyutf8 "utf8testingfile.txt"程序输出是这样的
Basic Latin: 49
Latin-1 Supplement: 18
Latin Extended-B: 9
Armenian: 9
Hebrew: 9
Arabic: 72
Enclosed Alphanumerics: 9
Old Persian: 9
chars count:184回答 1
Code Review用户
回答已采纳
发布于 2019-04-10 09:25:32
libmyutf8.c应该包括"libmyutf8.h"而不是"headerAll.h" --这可以确保函数定义与标头的原型一致。通过这种更改，不需要单独的"headerAll.h"，因此可以将其内联到"libmyutf8.h"中。
非公共函数应该使用静态链接来声明，这样才不会污染用户代码的命名空间。通过将static添加到签名中，我们可以防止其他代码可以使用相同的标识符但在将对象文件链接到一起时发现冲突的问题。
我们的函数应该接受指向const的指针，因为我们不打算修改输入字符串。
我建议在这里使用未签名的1u而不是1：
#定义CHECK_BIT(var，pos) ((var)和(1<<(Pos)
这将确保计算的所有术语都是无符号的，并且不会意外地提升到有符号类型(我不认为这在任何使用的地方都是一个问题，但它使推理变得更容易；在可能的情况下，总是更倾向于位操作的无符号类型)。
注释中的URI可以缩短：https://stackoverflow.com/q/523724。
当作为掩码操作进行测试时，测试一组比特会更简单。因此，与其：
} else if (CHECK\_BIT(currentByte, 7) && CHECK\_BIT(currentByte, 6) && CHECK\_BIT(currentByte, 5) && CHECK\_BIT(currentByte, 4) && !CHECK\_BIT(currentByte, 3))
我们可以写：
} else if ((currentByte & 0xf8) == 0xf0)我们可以使用二进制搜索，而不是increseCodeBlock中的线性搜索(这是increase的错误吗？)。另一种方法是使用字符的较高部分来索引到codePointBlocks中的起始点的指针表。我还没有完全想清楚，但会发生这样的事情：
/* Instead of writing this by hand, we could initialise this using
code to determine where each xx00 can be found */
static int blockIndex[] = {
0, /* 00xx - Basic Latin and Latin-1 */
2, /* 01xx - Latin Extended A and B */
3, /* 02xx - Latin B, to Spacing Modifiers */
6, /* 03xx - Combining Diacriticals, Greek, Coptic */
... /* lots more... */
};
static const size_t blockIndexSize = sizeof blockIndex / sizeof *blockIndex;
unsigned long highPart = codePoint / 0x100;
if (highPart >= blockIndexSize) {
highPart = blockIndexSize - 1;
}
for (int i = blockIndex[highPart]; i < codeBlocksCount; i++) {blockIndex表只允许我们在离目标更近的地方启动i，这样就可以节省我们对这么多条目的测试。
do/while在scanUtf8中的大循环看起来可能是一个for循环(我们有一个初始i = 0、一个i测试和一个增量++i，这样就可以更清楚地表达出来)。看起来像是
int i= 0；do { /*代码*/ ++i；} while (i < maxLength)；
大多数C程序员都会认为
for (int i = 0; i < maxLength; ++i) {
/* code */
}(对等确实需要0 < maxLength，因为这种更改将测试从结束移动到每个循环的开始)
与四个变量( byte1、byte2、byte3、byte4 )不同，最好是逐步构建代码点：
/* UNTESTED! */
/* Store next UTF-8 character into ch, and return next start position */
const char* scanUTF8(const char *s, wchar_t *ch)
{
int remaining = 0;
for (; *s; ++s) {
unsigned char c = *s;
if (remaining) {
/* check that it's a continuation byte */
if (c & 0xc0 != 0x80) {
*ch = BAD_UTF8;
return s;
}
*ch = (*ch << 6) + (c & 0x3f);
if (!--remaining) {
return s;
}
} else if (c & 0x80 == 0) {
/* single-byte (ASCII) */
*ch = c;
return s;
} else {
/* should be a start byte */
for (remaining = 3; remaining > 0; --remaining) {
if (~c >> (6 - remaining) == 1u) {
*ch = c & ((1u << (6 - remaining)) - 1);
break;
}
}
if (!remaining) {
/* not a valid start byte */
*ch = BAD_UTF8;
return ++s;
}
}
}
/* incomplete UTF-8 sequence */
*ch = BAD_UTF8;
return s;
}测试程序假设malloc()将成功，而不进行检查。不要这样做，即使在测试程序中也不行。实际上，特别是在测试程序中--让它以有用的消息失败，所以我们不把运行时错误与代码错误混淆起来。
这种性质的实用程序确实值得拥有一个体面的单元测试套件。虽然包括一个测试程序是很好的，但我们可以做得更好。使一组最小的输入和预期的输出在测试程序中具有以下优点：
- 它是独立的，而不需要单独的输入文件。
- 每个测试都使用代码的一个已知子集(使失败的测试更容易与其根本原因相关联)。
- 每个构建都可以自动运行测试(如果没有全部通过，测试就会失败)。
如果我们能够访问C++编译器，就可以使用用该语言实现的优秀测试框架之一(使用extern "C"很容易将我们的C函数链接到C++程序中)。
在编写单元测试时，我通常首先从简单的错误案例(空字符串、空字符串)开始。这将很快使我们进入测试思维。之后，我们可以开始添加成功案例和更复杂的错误(超出范围的字符、额外或缺失的连续字节、代理代码点、超长编码等等)。您可能需要阅读马库斯库恩译码器性能测试，以获得有关测试内容的一些想法。
页面原文内容由Code Review提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://codereview.stackexchange.com/questions/217169
复制相关文章
相似问题
腾讯云开发者
Copyright © 2013 - 2026 Tencent Cloud. All Rights Reserved. 腾讯云 版权所有
深圳市腾讯计算机系统有限公司 ICP备案/许可证号：粤B2-20090059 
粤公网安备44030502008569号
腾讯云计算（北京）有限责任公司 京ICP证150476号 | 京ICP备11018762号