diff --git a/README.md b/README.md index f9067497..924d9b49 100644 --- a/README.md +++ b/README.md @@ -2620,6 +2620,7 @@ Example: The following methods are available: TODO: document the API. +TODO: document the UTF8 API. ### M-CORE diff --git a/m-string.h b/m-string.h index 869d3766..5653446d 100644 --- a/m-string.h +++ b/m-string.h @@ -805,6 +805,198 @@ string_in_str(string_t v, FILE *f) return c == '"'; } +/* UTF8 Handling */ +typedef enum { + STRINGI_UTF8_STARTING = 0, + STRINGI_UTF8_DECODING_1 = 1, + STRINGI_UTF8_DECODING_2 = 2, + STRINGI_UTF8_DOCODING_3 = 3, + STRINGI_UTF8_ERROR = 4 +} stringi_utf8_state_e; + +typedef unsigned int string_unicode_t; + +/* UTF8 character classification: + * + * 0* --> type 1 byte A + * 10* --> chained byte B + * 110* --> type 2 byte C + * 1110* --> type 3 byte D + * 11110* --> type 4 byte E + * 111110* --> invalid I + */ +/* UTF8 State Transition table: + * ABCDEI + * +------ + * S|SI123III + * 1|ISIIIIII + * 2|I1IIIIII + * 3|I2IIIIII + * I|IIIIIIII + */ +/* The use of a string allows the compiler/linker to factorize it. */ +#define STRINGI_UTF8_STATE_TAB \ + "\000\004\001\002\003\004\004\004" \ + "\004\000\004\004\004\004\004\004" \ + "\004\001\004\004\004\004\004\004" \ + "\004\002\004\004\004\004\004\004" \ + "\004\004\004\004\004\004\004\004" + +/* Main generic UTF8 decoder + It shall be (nearly) branchless on any CPU. + It takes a character, and the previous state and the previous value of the unicode value. + It updates the state and the decoded unicode value. + A decoded unicoded value is valid only when the state is STARTING. + */ +static inline void stringi_utf8_decode(char c, stringi_utf8_state_e *state, + string_unicode_t *unicode) +{ + const int type = m_core_clz((unsigned char)~c) - (sizeof(unsigned long) - 1) * CHAR_BIT; + const string_unicode_t mask1 = -(string_unicode_t)(*state != STRINGI_UTF8_STARTING); + const string_unicode_t mask2 = (0xFFU >> type); + *unicode = ((*unicode << 6) & mask1) | (c & mask2); + *state = STRINGI_UTF8_STATE_TAB[*state * 8 + type]; +} + +/* Check if the given array of characters is a valid UTF8 stream */ +/* NOTE: Non-canonical representation are not rejected */ +static inline bool stringi_utf8_valid_str_p(const char str[]) +{ + stringi_utf8_state_e s = STRINGI_UTF8_STARTING; + string_unicode_t u = 0; + while (*str) { + stringi_utf8_decode(*str, &s, &u); + if ((s == STRINGI_UTF8_ERROR) + ||(s == STRINGI_UTF8_STARTING + &&(u > 0x10FFFF /* out of range */ + ||(u >= 0xD800 && u <= 0xDFFF) /* surrogate halves */))) + return false; + str++; + } + return true; +} + +/* Computer the number of unicode characters are represented in the UTF8 stream */ +static inline size_t stringi_utf8_length(const char str[]) +{ + size_t size = 0; + stringi_utf8_state_e s = STRINGI_UTF8_STARTING; + string_unicode_t u = 0; + while (*str) { + stringi_utf8_decode(*str, &s, &u); + if (s == STRINGI_UTF8_ERROR) return -1; + if (s == STRINGI_UTF8_STARTING) size++; + str++; + } + return size; +} + +/* Encode an unicode into an UTF8 stream of characters */ +static inline int stringi_utf8_encode(char buffer[5], string_unicode_t u) +{ + if (M_LIKELY (u <= 0x7F)) { + buffer[0] = u; + buffer[1] = 0; + return 1; + } else if (u <= 0x7FF) { + buffer[0] = 0xC0 | (u >> 6); + buffer[1] = 0x80 | (u & 0x3F); + buffer[2] = 0; + return 2; + } else if (u <= 0xFFFF) { + buffer[0] = 0xE0 | (u >> 12); + buffer[1] = 0x80 | ((u >> 6) & 0x3F); + buffer[2] = 0x80 | (u & 0x3F); + buffer[3] = 0; + return 3; + } else { + buffer[0] = 0xF0 | (u >> 18); + buffer[1] = 0x80 | ((u >> 12) & 0x3F); + buffer[2] = 0x80 | ((u >> 6) & 0x3F); + buffer[3] = 0x80 | (u & 0x3F); + buffer[4] = 0; + return 4; + } +} + +/* Iterator on a string over UTF8 encoded characters */ +typedef struct { + string_unicode_t u; + const char *ptr; + const char *next_ptr; +} string_it_t[1]; + +/* Start iteration over the UTF8 encoded unicode value */ +static inline void +string_it(string_it_t it, const string_t str) +{ + STRING_CONTRACT(str); + assert(it != NULL); + it->ptr = str->ptr; + it->next_ptr = str->ptr; + it->u = 0; +} + +static inline bool +string_end_p (string_it_t it) +{ + assert (it != NULL); + if (M_UNLIKELY (*it->ptr == 0)) + return true; + stringi_utf8_state_e state = STRINGI_UTF8_STARTING; + string_unicode_t u = 0; + const char *str = it->ptr; + do { + stringi_utf8_decode(*str, &state, &u); + str++; + } while (state != STRINGI_UTF8_STARTING && state != STRINGI_UTF8_ERROR && *str != 0); + it->next_ptr = str; + it->u = M_UNLIKELY (state == STRINGI_UTF8_ERROR) ? -1U : u; + return false; +} + +static inline void +string_next (string_it_t it) +{ + assert (it != NULL); + it->ptr = it->next_ptr; +} + +static inline string_unicode_t +string_get_cref (const string_it_t it) +{ + assert (it != NULL); + return it->u; +} + +/* Push unicode into string, encoding it in UTF8 */ +static inline void +string_push_u (string_t str, string_unicode_t u) +{ + STRING_CONTRACT(str); + char buffer[4+1]; + stringi_utf8_encode(buffer, u); + string_cat_str(str, buffer); +} + +/* Compute the length in UTF8 characters in the string */ +static inline size_t +string_length_u(string_t str) +{ + STRING_CONTRACT(str); + return stringi_utf8_length(str->ptr); +} + +/* Check if a string is a valid UTF8 encoded stream */ +static inline bool +string_utf8_p(string_t str) +{ + STRING_CONTRACT(str); + return stringi_utf8_valid_str_p(str->ptr); +} + + +/* Define the split function for algorithm inclusion */ #define STRING_SPLIT(name, oplist, type_oplist) \ static inline void M_C(name, _split)(M_GET_TYPE oplist cont, \ const string_t str, const char sep) \ @@ -843,6 +1035,7 @@ string_in_str(string_t v, FILE *f) } \ } \ + /* Use of Compound Literals to init a constant string. NOTE: The use of the additional structure layer is to ensure that the pointer to char is properly aligned to an int (this @@ -895,6 +1088,8 @@ namespace m_string { STRING_DECL_INIT(v); \ string_printf (v, format, __VA_ARGS__) + +/* Define the OPLIST of a STRING */ #define STRING_OPLIST \ (INIT(string_init),INIT_SET(string_init_set), SET(string_set), \ INIT_MOVE(string_init_move), MOVE(string_move), \ @@ -904,9 +1099,10 @@ namespace m_string { OUT_STR(string_out_str), IN_STR(string_in_str), \ EXT_ALGO(STRING_SPLIT) \ ) - +/* Register the OPLIST as a global one */ #define M_OPL_string_t() STRING_OPLIST + /* Macro encapsulation to give default value for start offset */ #define string_search_char(v, ...) \ M_APPLY(string_search_char, v, M_IF_DEFAULT1(0, __VA_ARGS__)) diff --git a/tests/test-mstring.c b/tests/test-mstring.c index f3c028bb..88fa549e 100644 --- a/tests/test-mstring.c +++ b/tests/test-mstring.c @@ -32,6 +32,81 @@ static void test1(void) } } +static void test_utf8_basic(void) +{ + string_t s; + string_init (s); + + /* Test internal encode + decode for all characters */ + for (string_unicode_t i = 1; i < 0x10ffff; i++) { + if (i < 0xD800U || i > 0xDFFFU) { + string_it_t it; + char buf[5] = {0}; + stringi_utf8_encode (buf, i); + /* Low level access for internal testing */ + it->ptr = buf; + bool b = string_end_p(it); + assert (b == false); + assert (it->u == i); + assert (string_get_cref(it) == i); + assert (*it->next_ptr == 0); + assert (stringi_utf8_length(buf) == 1); + assert (stringi_utf8_valid_str_p(buf) == true); + /* Higher level access */ + string_clean(s); + string_push_u(s, i); + assert(string_utf8_p(s) == true); + assert(string_length_u(s) == 1); + } + } + + /* Reject out of range value */ + for (string_unicode_t i = 0x110000; i < 0x1fffff; i++) { + string_clean(s); + string_push_u(s, i); + assert(string_utf8_p(s) == false); + } + /* Test rejection of surrogate halves */ + for (string_unicode_t i = 0xd800; i <= 0xdfff; i++) { + string_clean(s); + string_push_u(s, i); + assert(string_utf8_p(s) == false); + } + /* NOTE: Non-canonical representation are not rejected */ + string_clear(s); +} + +static void test_utf8_it(void) +{ + string_t s; + string_init(s); + const unsigned int tab[] = { 45, 1458, 25623, 129, 24 }; + for(int i = 0; i < 5; i++) + string_push_u (s, tab[i]); + assert (string_length_u(s) == 5); + string_it_t it; + int i = 0; + for(string_it(it, s) ; !string_end_p(it); string_next(it), i++) { + assert (i < 5); + assert (string_get_cref(it) == tab[i]); + } + + string_set_str (s, "H€llo René Chaînôr¬"); + assert(string_length_u(s) == 19); + string_unicode_t tab2[19]; + i = 0; + for(string_it(it, s) ; !string_end_p(it); string_next(it), i++) { + assert (i < 19); + tab2[i] = string_get_cref(it); + } + assert (i == 19); + string_clean(s); + for(i = 0 ; i < 19; i++) + string_push_u(s, tab2[i]); + assert (string_equal_str_p(s, "H€llo René Chaînôr¬")); + + string_clear(s); +} int main(void) { @@ -341,5 +416,7 @@ int main(void) string_clear (s1); string_clear (s2); + test_utf8_basic(); + test_utf8_it(); exit(0); }