From f3efaf951fc00a590eed5497d52aca92230f3da5 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 15:57:32 +0100
Subject: [PATCH 01/19] Add first functions to handle UTF8 strings.

---
 m-string.h | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/m-string.h b/m-string.h
index 869d3766..ead313dd 100644
--- a/m-string.h
+++ b/m-string.h
@@ -805,6 +805,83 @@ string_in_str(string_t v, FILE *f)
   return c == '"';
 }
 
+/* UTF8 Handling */
+typedef enum {
+  STRINGI_UTF8_STARTING = 0,
+  STRINGI_UTF8_DECODING_1 = 1,
+  STRINGI_UTF8_DECODING_2 = 2,
+  STRINGI_UTF8_DOCODING_3 = 3,
+  STRINGI_UTF8_ERROR = 4  
+} stringi_utf8_state_e;
+
+typedef unsigned int string_unicode_t;
+
+/* UTF8 character classification:
+ * 
+ * 0*       --> type 1 byte  A
+ * 10*      --> chained byte B
+ * 110*     --> type 2 byte  C
+ * 1110*    --> type 3 byte  D
+ * 11110*   --> type 4 byte  E
+ * 111110*  --> invalid      I
+ */
+/* UTF8 State Transition table:
+ *    ABCDEI
+ *   +------
+ *  S|SI123III
+ *  1|ISIIIIII
+ *  2|I1IIIIII
+ *  3|I2IIIIII
+ *  I|IIIIIIII
+ */
+#define STRINGI_UTF8_STATE_TAB                                          \
+  "\000\004\001\002\003\004\004\004"                                    \
+  "\004\000\004\004\004\004\004\004"                                    \
+  "\004\001\004\004\004\004\004\004"                                    \
+  "\004\002\004\004\004\004\004\004"                                    \
+  "\004\004\004\004\004\004\004\004"
+
+static inline void stringi_utf8_decode(char car, stringi_utf8_state_e *state,
+                                       string_unicode_t *unicode)
+{
+  const int type = m_core_clz((unsigned char)~car) - (sizeof(unsigned long) - 1) * CHAR_BIT;
+  const string_unicode_t mask1 = -(string_unicode_t)(*state != STRINGI_UTF8_STARTING);
+  const string_unicode_t mask2 = (0xFFU >> type);
+  *unicode = ((*unicode << 6) & mask1) | (car & mask2);
+  *state = STRINGI_UTF8_STATE_TAB[*state * 8 + type];
+}
+
+static inline bool stringi_utf8_valid_str_p(const char str[])
+{
+  stringi_utf8_state_e s = STRINGI_UTF8_STARTING;
+  string_unicode_t u = 0;
+  while (*str) {
+    stringi_utf8_decode(*str, &s, &u);
+    if ((s == STRINGI_UTF8_ERROR)
+        ||(s == STRINGI_UTF8_STARTING
+           &&(u > 0x10FFFF /* out of range */
+              ||(u >= 0xD800 && u <= 0xDFFF) /* surrogate halves */)))
+      return false;
+    str++;
+  }
+  return true;
+}
+
+static inline size_t stringi_utf8_length(const char str[])
+{
+  size_t size = 0;
+  stringi_utf8_state_e s = STRINGI_UTF8_STARTING;
+  string_unicode_t u = 0;
+  while (*str) {
+    stringi_utf8_decode(*str, &s, &u);
+    if (s == STRINGI_UTF8_ERROR) return -1;
+    if (s == STRINGI_UTF8_STARTING) size++;
+    str++;
+  }
+  return size;
+}
+
+
 #define STRING_SPLIT(name, oplist, type_oplist)                         \
   static inline void M_C(name, _split)(M_GET_TYPE oplist cont,          \
                                    const string_t str, const char sep)  \

From c7516866400a903884b711dc30cf93e28e66a97d Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 15:58:24 +0100
Subject: [PATCH 02/19] Update comments

---
 m-string.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/m-string.h b/m-string.h
index ead313dd..4d323f1f 100644
--- a/m-string.h
+++ b/m-string.h
@@ -972,6 +972,7 @@ namespace m_string {
   STRING_DECL_INIT(v);                                                  \
   string_printf (v, format, __VA_ARGS__)
 
+/* Define the OPLIST of a STRING */
 #define STRING_OPLIST                                                   \
   (INIT(string_init),INIT_SET(string_init_set), SET(string_set),        \
    INIT_MOVE(string_init_move), MOVE(string_move),                      \
@@ -981,7 +982,7 @@ namespace m_string {
    OUT_STR(string_out_str), IN_STR(string_in_str),                      \
    EXT_ALGO(STRING_SPLIT)                                               \
    )
-
+/* Register the OPLIST as a global one */
 #define M_OPL_string_t() STRING_OPLIST
 
 /* Macro encapsulation to give default value for start offset */

From b5a0d3dd0fa5e9b3ce855a37cbc6ab07da71c8e4 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 15:58:46 +0100
Subject: [PATCH 03/19] Reformat

---
 m-string.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/m-string.h b/m-string.h
index 4d323f1f..24054e90 100644
--- a/m-string.h
+++ b/m-string.h
@@ -920,6 +920,7 @@ static inline size_t stringi_utf8_length(const char str[])
     }                                                                   \
   }                                                                     \
 
+
 /* Use of Compound Literals to init a constant string.
    NOTE: The use of the additional structure layer is to ensure
    that the pointer to char is properly aligned to an int (this
@@ -972,6 +973,7 @@ namespace m_string {
   STRING_DECL_INIT(v);                                                  \
   string_printf (v, format, __VA_ARGS__)
 
+
 /* Define the OPLIST of a STRING */
 #define STRING_OPLIST                                                   \
   (INIT(string_init),INIT_SET(string_init_set), SET(string_set),        \
@@ -985,6 +987,7 @@ namespace m_string {
 /* Register the OPLIST as a global one */
 #define M_OPL_string_t() STRING_OPLIST
 
+
 /* Macro encapsulation to give default value for start offset */
 #define string_search_char(v, ...)					\
   M_APPLY(string_search_char, v, M_IF_DEFAULT1(0, __VA_ARGS__))

From ccb6fd6fb26719e6958a3576b2299959a2401525 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:35:27 +0100
Subject: [PATCH 04/19] Add new function.

---
 m-string.h | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/m-string.h b/m-string.h
index 24054e90..3a6ca47c 100644
--- a/m-string.h
+++ b/m-string.h
@@ -881,6 +881,77 @@ static inline size_t stringi_utf8_length(const char str[])
   return size;
 }
 
+typedef struct {
+  string_unicode_t u;
+  const char *ptr;
+  const char *next_ptr;
+} string_it_t[1];
+
+static inline void
+string_it(string_it_t it, const string_t str)
+{
+  it->ptr    = str->ptr;
+  it->next_ptr  = str->ptr;
+  it->u      = 0;
+}
+
+static inline bool
+string_end_p (string_it_t it)
+{
+  if (*it->ptr == 0)
+    return true;
+  stringi_utf8_state_e state =  STRINGI_UTF8_STARTING;
+  string_unicode_t u = 0;
+  const char *str = it->ptr;
+  do {
+    stringi_utf8_decode(*str, &state, &u);
+    str++;
+  } while (state != STRINGI_UTF8_STARTING && state != STRINGI_UTF8_ERROR && *str != 0);
+  it->next_ptr = str;
+  it->u = M_UNLIKELY (state == STRINGI_UTF8_ERROR) ? -1U : u;
+  return false;
+}
+
+static inline void
+string_next (string_it_t it)
+{
+  it->ptr = it->next_ptr;
+}
+
+static inline string_unicode_t
+string_get_cref (const string_it_t it)
+{
+  return it->u;
+}
+
+/* Push unicode into string, encoding it in UTF8 */
+static inline void
+string_push_u (string_t str, string_unicode_t u)
+{
+  char buffer[4+1];
+  if (M_LIKELY (u <= 0x7F)) {
+    buffer[0] = u;
+    buffer[1] = 0;
+  } else if (u <= 0x7FF) {
+    buffer[0] = 0xC0 | (u >> 6);
+    buffer[1] = 0x80 | (u & 0x3F);
+    buffer[2] = 0;
+  } else if (u <= 0xFFFF) {
+    buffer[0] = 0xE0 | (u >> 12);
+    buffer[1] = 0x80 | ((u >> 6) & 0x3F);
+    buffer[2] = 0x80 | (u & 0x3F);
+    buffer[3] = 0;
+  } else {
+    buffer[0] = 0xF0 | (u >> 18);
+    buffer[1] = 0x80 | ((u >> 12) & 0x3F);
+    buffer[2] = 0x80 | ((u >> 6) & 0x3F);
+    buffer[3] = 0x80 | (u & 0x3F);
+    buffer[4] = 0;
+  }
+  string_cat_str(str, buffer);
+}
+
+
 
 #define STRING_SPLIT(name, oplist, type_oplist)                         \
   static inline void M_C(name, _split)(M_GET_TYPE oplist cont,          \

From 5f98cc1eb6c88098397cf627a50032d25de6488e Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:36:46 +0100
Subject: [PATCH 05/19] Factorisation

---
 m-string.h | 43 ++++++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/m-string.h b/m-string.h
index 3a6ca47c..85c83a56 100644
--- a/m-string.h
+++ b/m-string.h
@@ -881,6 +881,29 @@ static inline size_t stringi_utf8_length(const char str[])
   return size;
 }
 
+static inline void stringi_utf8_encode(char buffer[5], string_unicode_t u)
+{
+  if (M_LIKELY (u <= 0x7F)) {
+    buffer[0] = u;
+    buffer[1] = 0;
+  } else if (u <= 0x7FF) {
+    buffer[0] = 0xC0 | (u >> 6);
+    buffer[1] = 0x80 | (u & 0x3F);
+    buffer[2] = 0;
+  } else if (u <= 0xFFFF) {
+    buffer[0] = 0xE0 | (u >> 12);
+    buffer[1] = 0x80 | ((u >> 6) & 0x3F);
+    buffer[2] = 0x80 | (u & 0x3F);
+    buffer[3] = 0;
+  } else {
+    buffer[0] = 0xF0 | (u >> 18);
+    buffer[1] = 0x80 | ((u >> 12) & 0x3F);
+    buffer[2] = 0x80 | ((u >> 6) & 0x3F);
+    buffer[3] = 0x80 | (u & 0x3F);
+    buffer[4] = 0;
+  }
+}
+
 typedef struct {
   string_unicode_t u;
   const char *ptr;
@@ -929,25 +952,7 @@ static inline void
 string_push_u (string_t str, string_unicode_t u)
 {
   char buffer[4+1];
-  if (M_LIKELY (u <= 0x7F)) {
-    buffer[0] = u;
-    buffer[1] = 0;
-  } else if (u <= 0x7FF) {
-    buffer[0] = 0xC0 | (u >> 6);
-    buffer[1] = 0x80 | (u & 0x3F);
-    buffer[2] = 0;
-  } else if (u <= 0xFFFF) {
-    buffer[0] = 0xE0 | (u >> 12);
-    buffer[1] = 0x80 | ((u >> 6) & 0x3F);
-    buffer[2] = 0x80 | (u & 0x3F);
-    buffer[3] = 0;
-  } else {
-    buffer[0] = 0xF0 | (u >> 18);
-    buffer[1] = 0x80 | ((u >> 12) & 0x3F);
-    buffer[2] = 0x80 | ((u >> 6) & 0x3F);
-    buffer[3] = 0x80 | (u & 0x3F);
-    buffer[4] = 0;
-  }
+  stringi_utf8_encode(buffer, u);
   string_cat_str(str, buffer);
 }
 

From f1ed954dd06be770ae741e588ede80c0d7c41680 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:38:59 +0100
Subject: [PATCH 06/19] Add functions

---
 m-string.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/m-string.h b/m-string.h
index 85c83a56..70b0b57a 100644
--- a/m-string.h
+++ b/m-string.h
@@ -956,6 +956,17 @@ string_push_u (string_t str, string_unicode_t u)
   string_cat_str(str, buffer);
 }
 
+static inline size_t
+string_length_u(string_t str)
+{
+  return stringi_utf8_length(str->ptr);
+}
+
+static inline bool
+string_utf8_p(string_t str)
+{
+  return stringi_utf8_valid_str_p(str->ptr);
+}
 
 
 #define STRING_SPLIT(name, oplist, type_oplist)                         \

From 417571755ede364cdfddee2f6425e9e1394a4ed9 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:39:28 +0100
Subject: [PATCH 07/19] Add assertions

---
 m-string.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/m-string.h b/m-string.h
index 70b0b57a..90f8d04f 100644
--- a/m-string.h
+++ b/m-string.h
@@ -913,6 +913,7 @@ typedef struct {
 static inline void
 string_it(string_it_t it, const string_t str)
 {
+  STRING_CONTRACT(str);
   it->ptr    = str->ptr;
   it->next_ptr  = str->ptr;
   it->u      = 0;
@@ -951,20 +952,24 @@ string_get_cref (const string_it_t it)
 static inline void
 string_push_u (string_t str, string_unicode_t u)
 {
+  STRING_CONTRACT(str);
   char buffer[4+1];
   stringi_utf8_encode(buffer, u);
   string_cat_str(str, buffer);
 }
 
+/* Compute the length in UTF8 characters in the string */
 static inline size_t
 string_length_u(string_t str)
 {
+  STRING_CONTRACT(str);
   return stringi_utf8_length(str->ptr);
 }
 
 static inline bool
 string_utf8_p(string_t str)
 {
+  STRING_CONTRACT(str);
   return stringi_utf8_valid_str_p(str->ptr);
 }
 

From d0ecc03abe4b658f671898985d4ea9d2d124d5e8 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:39:58 +0100
Subject: [PATCH 08/19] Add some more assertions

---
 m-string.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/m-string.h b/m-string.h
index 90f8d04f..e2f31489 100644
--- a/m-string.h
+++ b/m-string.h
@@ -922,6 +922,7 @@ string_it(string_it_t it, const string_t str)
 static inline bool
 string_end_p (string_it_t it)
 {
+  assert (it != NULL);
   if (*it->ptr == 0)
     return true;
   stringi_utf8_state_e state =  STRINGI_UTF8_STARTING;
@@ -939,12 +940,14 @@ string_end_p (string_it_t it)
 static inline void
 string_next (string_it_t it)
 {
+  assert (it != NULL);
   it->ptr = it->next_ptr;
 }
 
 static inline string_unicode_t
 string_get_cref (const string_it_t it)
 {
+  assert (it != NULL);
   return it->u;
 }
 

From 5cbae75ed03decfd5c8fad66e3d8878636d2562f Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:54:04 +0100
Subject: [PATCH 09/19] Update

---
 m-string.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/m-string.h b/m-string.h
index e2f31489..8954285e 100644
--- a/m-string.h
+++ b/m-string.h
@@ -851,6 +851,7 @@ static inline void stringi_utf8_decode(char car, stringi_utf8_state_e *state,
   *state = STRINGI_UTF8_STATE_TAB[*state * 8 + type];
 }
 
+/* NOTE: Non-canonical representation are not rejected */
 static inline bool stringi_utf8_valid_str_p(const char str[])
 {
   stringi_utf8_state_e s = STRINGI_UTF8_STARTING;

From 873e1710c9003485f48b0a959abb2a95bf24f7c0 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:54:11 +0100
Subject: [PATCH 10/19] Add tests

---
 tests/test-mstring.c | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/tests/test-mstring.c b/tests/test-mstring.c
index f3c028bb..469688c7 100644
--- a/tests/test-mstring.c
+++ b/tests/test-mstring.c
@@ -32,6 +32,48 @@ static void test1(void)
   }
 }
 
+static void test_utf8_basic(void)
+{
+  string_t s;
+  string_init (s);
+  
+  /* Test internal encode + decode for all characters */
+  for (string_unicode_t i = 0; i < 0x10ffff; i++) {
+    if (i < 0xD800U || i > 0xDFFFU) {
+      string_it_t it;
+      char buf[5] = {0};
+      stringi_utf8_encode (buf, i);
+      /* Low level access for internal testing */
+      it->ptr = buf;
+      bool b = string_end_p(it);
+      assert (b == false);
+      assert (it->u == i);
+      assert (*it->next_ptr == 0);
+      assert (stringi_utf8_length(buf) == 1);
+      assert (stringi_utf8_valid_str_p(buf) == true);
+      /* Higher level access */
+      string_clean(s);
+      string_push_u(s, i);
+      assert(string_utf8_p(s) == true);
+      assert(string_length_u(s) == 1); 
+    }
+  }
+  
+  /* Reject out of range value */
+  for (string_unicode_t i = 0x110000; i < 0x1fffff; i++) {
+    string_clean(s);
+    string_push_u(s, i);
+    assert(string_utf8_p(s) == false);
+  }
+  /* Test rejection of surrogate halves */
+  for (string_unicode_t i = 0xd800; i <= 0xdfff; i++) {
+    string_clean(s);
+    string_push_u(s, i);
+    assert(string_utf8_p(s) == false);
+  }
+  /* NOTE: Non-canonical representation are not rejected */
+  string_clear(s);
+}
 
 int main(void)
 {
@@ -341,5 +383,6 @@ int main(void)
   string_clear (s1);
   string_clear (s2);
   
+  test_utf8_basic();
   exit(0);
 }

From ca5263b4889121a176902c4dae44ea9267398f39 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 16:57:55 +0100
Subject: [PATCH 11/19] Update comments

---
 m-string.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/m-string.h b/m-string.h
index 8954285e..366723e7 100644
--- a/m-string.h
+++ b/m-string.h
@@ -841,6 +841,9 @@ typedef unsigned int string_unicode_t;
   "\004\002\004\004\004\004\004\004"                                    \
   "\004\004\004\004\004\004\004\004"
 
+/* Main generic UTF8 decoder
+   It shall be (nearly) branchless on any CPU
+ */
 static inline void stringi_utf8_decode(char car, stringi_utf8_state_e *state,
                                        string_unicode_t *unicode)
 {
@@ -905,6 +908,7 @@ static inline void stringi_utf8_encode(char buffer[5], string_unicode_t u)
   }
 }
 
+/* Iterator on a string over UTF8 encoded characters */
 typedef struct {
   string_unicode_t u;
   const char *ptr;

From c4447123f3c15180dc5aa538f4b6c07ca5b0b556 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 18:11:49 +0100
Subject: [PATCH 12/19] Add more tests

---
 tests/test-mstring.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/test-mstring.c b/tests/test-mstring.c
index 469688c7..dfbf06d0 100644
--- a/tests/test-mstring.c
+++ b/tests/test-mstring.c
@@ -75,6 +75,24 @@ static void test_utf8_basic(void)
   string_clear(s);
 }
 
+static void test_utf8_it(void)
+{
+  string_t s;
+  string_init(s);
+  const unsigned int tab[] = { 45, 1458, 25623, 129, 24 };
+  for(int i = 0; i < 5; i++)
+    string_push_u (s, i);
+  string_it_t it;
+  int i = 0;
+  for(string_it(it, s) ; !string_end_p(it); string_next(it), i++) {
+    assert (i < 5);
+    assert (string_get_cref(it) == tab[i]);
+  }
+  string_set_str (s, "H€llo René Chaînôr¬");
+  assert(string_length_u(s) == 19);
+  string_clear(s);
+}
+
 int main(void)
 {
   string_t s1;
@@ -384,5 +402,6 @@ int main(void)
   string_clear (s2);
   
   test_utf8_basic();
+  test_utf8_it();
   exit(0);
 }

From a24464cfedbf7155c01656f6bafb0b894960cd10 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Sat, 9 Dec 2017 18:11:58 +0100
Subject: [PATCH 13/19] Return size of the encoding (in case of)

---
 m-string.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/m-string.h b/m-string.h
index 366723e7..d6dfd62b 100644
--- a/m-string.h
+++ b/m-string.h
@@ -885,26 +885,30 @@ static inline size_t stringi_utf8_length(const char str[])
   return size;
 }
 
-static inline void stringi_utf8_encode(char buffer[5], string_unicode_t u)
+static inline int stringi_utf8_encode(char buffer[5], string_unicode_t u)
 {
   if (M_LIKELY (u <= 0x7F)) {
     buffer[0] = u;
     buffer[1] = 0;
+    return 1;
   } else if (u <= 0x7FF) {
     buffer[0] = 0xC0 | (u >> 6);
     buffer[1] = 0x80 | (u & 0x3F);
     buffer[2] = 0;
+    return 2;
   } else if (u <= 0xFFFF) {
     buffer[0] = 0xE0 | (u >> 12);
     buffer[1] = 0x80 | ((u >> 6) & 0x3F);
     buffer[2] = 0x80 | (u & 0x3F);
     buffer[3] = 0;
+    return 3;
   } else {
     buffer[0] = 0xF0 | (u >> 18);
     buffer[1] = 0x80 | ((u >> 12) & 0x3F);
     buffer[2] = 0x80 | ((u >> 6) & 0x3F);
     buffer[3] = 0x80 | (u & 0x3F);
     buffer[4] = 0;
+    return 4;
   }
 }
 
@@ -928,7 +932,7 @@ static inline bool
 string_end_p (string_it_t it)
 {
   assert (it != NULL);
-  if (*it->ptr == 0)
+  if (M_UNLIKELY (*it->ptr == 0))
     return true;
   stringi_utf8_state_e state =  STRINGI_UTF8_STARTING;
   string_unicode_t u = 0;

From df73de9e10c229727d43474f66bb9ae9d4608a7f Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:41:42 +0100
Subject: [PATCH 14/19] Add comments.

---
 m-string.h | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/m-string.h b/m-string.h
index d6dfd62b..a68ad6a9 100644
--- a/m-string.h
+++ b/m-string.h
@@ -834,6 +834,7 @@ typedef unsigned int string_unicode_t;
  *  3|I2IIIIII
  *  I|IIIIIIII
  */
+/* The use of a string allows the compiler/linker to factorize it. */
 #define STRINGI_UTF8_STATE_TAB                                          \
   "\000\004\001\002\003\004\004\004"                                    \
   "\004\000\004\004\004\004\004\004"                                    \
@@ -842,18 +843,22 @@ typedef unsigned int string_unicode_t;
   "\004\004\004\004\004\004\004\004"
 
 /* Main generic UTF8 decoder
-   It shall be (nearly) branchless on any CPU
+   It shall be (nearly) branchless on any CPU.
+   It takes a character, and the previous state and the previous value of the unicode value.
+   It updates the state and the decoded unicode value.
+   A decoded unicoded value is valid only when the state is STARTING.
  */
-static inline void stringi_utf8_decode(char car, stringi_utf8_state_e *state,
+static inline void stringi_utf8_decode(char c, stringi_utf8_state_e *state,
                                        string_unicode_t *unicode)
 {
-  const int type = m_core_clz((unsigned char)~car) - (sizeof(unsigned long) - 1) * CHAR_BIT;
+  const int type = m_core_clz((unsigned char)~c) - (sizeof(unsigned long) - 1) * CHAR_BIT;
   const string_unicode_t mask1 = -(string_unicode_t)(*state != STRINGI_UTF8_STARTING);
   const string_unicode_t mask2 = (0xFFU >> type);
-  *unicode = ((*unicode << 6) & mask1) | (car & mask2);
+  *unicode = ((*unicode << 6) & mask1) | (c & mask2);
   *state = STRINGI_UTF8_STATE_TAB[*state * 8 + type];
 }
 
+/* Check if the given array of characters is a valid UTF8 stream */
 /* NOTE: Non-canonical representation are not rejected */
 static inline bool stringi_utf8_valid_str_p(const char str[])
 {
@@ -871,6 +876,7 @@ static inline bool stringi_utf8_valid_str_p(const char str[])
   return true;
 }
 
+/* Computer the number of unicode characters are represented in the UTF8 stream */
 static inline size_t stringi_utf8_length(const char str[])
 {
   size_t size = 0;
@@ -885,6 +891,7 @@ static inline size_t stringi_utf8_length(const char str[])
   return size;
 }
 
+/* Encode an unicode into an UTF8 stream of characters */
 static inline int stringi_utf8_encode(char buffer[5], string_unicode_t u)
 {
   if (M_LIKELY (u <= 0x7F)) {
@@ -919,13 +926,15 @@ typedef struct {
   const char *next_ptr;
 } string_it_t[1];
 
+/* Start iteration over the UTF8 encoded unicode value */
 static inline void
 string_it(string_it_t it, const string_t str)
 {
   STRING_CONTRACT(str);
-  it->ptr    = str->ptr;
-  it->next_ptr  = str->ptr;
-  it->u      = 0;
+  assert(it != NULL);
+  it->ptr      = str->ptr;
+  it->next_ptr = str->ptr;
+  it->u        = 0;
 }
 
 static inline bool
@@ -978,6 +987,7 @@ string_length_u(string_t str)
   return stringi_utf8_length(str->ptr);
 }
 
+/* Check if a string is a valid UTF8 encoded stream */
 static inline bool
 string_utf8_p(string_t str)
 {

From 48828f085b6f752f655008dff828b4c38a8be276 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:43:27 +0100
Subject: [PATCH 15/19] Add comment

---
 m-string.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/m-string.h b/m-string.h
index a68ad6a9..5653446d 100644
--- a/m-string.h
+++ b/m-string.h
@@ -996,6 +996,7 @@ string_utf8_p(string_t str)
 }
 
 
+/* Define the split function for algorithm inclusion */
 #define STRING_SPLIT(name, oplist, type_oplist)                         \
   static inline void M_C(name, _split)(M_GET_TYPE oplist cont,          \
                                    const string_t str, const char sep)  \

From cc30ecebfcc6cfe68745f03c7b4b28bfcd087967 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:51:16 +0100
Subject: [PATCH 16/19] Fix tests.

---
 tests/test-mstring.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test-mstring.c b/tests/test-mstring.c
index dfbf06d0..70caafaa 100644
--- a/tests/test-mstring.c
+++ b/tests/test-mstring.c
@@ -38,7 +38,7 @@ static void test_utf8_basic(void)
   string_init (s);
   
   /* Test internal encode + decode for all characters */
-  for (string_unicode_t i = 0; i < 0x10ffff; i++) {
+  for (string_unicode_t i = 1; i < 0x10ffff; i++) {
     if (i < 0xD800U || i > 0xDFFFU) {
       string_it_t it;
       char buf[5] = {0};
@@ -48,6 +48,7 @@ static void test_utf8_basic(void)
       bool b = string_end_p(it);
       assert (b == false);
       assert (it->u == i);
+      assert (string_get_cref(it) == i);
       assert (*it->next_ptr == 0);
       assert (stringi_utf8_length(buf) == 1);
       assert (stringi_utf8_valid_str_p(buf) == true);
@@ -81,7 +82,8 @@ static void test_utf8_it(void)
   string_init(s);
   const unsigned int tab[] = { 45, 1458, 25623, 129, 24 };
   for(int i = 0; i < 5; i++)
-    string_push_u (s, i);
+    string_push_u (s, tab[i]);
+  assert (string_length_u(s) == 5);
   string_it_t it;
   int i = 0;
   for(string_it(it, s) ; !string_end_p(it); string_next(it), i++) {

From 6d3d2c9eac11a26bf63826b9aa7384b60499d4c8 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:52:25 +0100
Subject: [PATCH 17/19] Update doc

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index c10ed61f..6eb2bccb 100644
--- a/README.md
+++ b/README.md
@@ -2619,6 +2619,7 @@ Example:
 The following methods are available:
 
 TODO: document the API.
+TODO: document the UTF8 API.
 
 ### M-CORE
 

From 3296c5313c1ae5ba09f0fa4786141d41eacde939 Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:56:41 +0100
Subject: [PATCH 18/19] Add more tests.

---
 tests/test-mstring.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test-mstring.c b/tests/test-mstring.c
index 70caafaa..8fafc143 100644
--- a/tests/test-mstring.c
+++ b/tests/test-mstring.c
@@ -92,6 +92,18 @@ static void test_utf8_it(void)
   }
   string_set_str (s, "H€llo René Chaînôr¬");
   assert(string_length_u(s) == 19);
+  string_unicode_t tab2[19];
+  i = 0;
+  for(string_it(it, s) ; !string_end_p(it); string_next(it), i++) {
+    assert (i < 19);
+    tab2[i] = string_get_cref(it);
+  }
+  assert (i == 19);
+  string_clean(s);
+  for(i = 0 ; i < 19; i++)
+    string_push_u(s, tab2[i]);
+  assert (string_equal_str_p(s, "H€llo René Chaînôr¬"));
+
   string_clear(s);
 }
 

From eae0a3833a89a84606250e250cdd0dd6bf2e2dbe Mon Sep 17 00:00:00 2001
From: Patrick Pelissier <patrick.pelissier@gmail.com>
Date: Wed, 13 Dec 2017 21:57:24 +0100
Subject: [PATCH 19/19] Add separation

---
 tests/test-mstring.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test-mstring.c b/tests/test-mstring.c
index 8fafc143..88fa549e 100644
--- a/tests/test-mstring.c
+++ b/tests/test-mstring.c
@@ -90,6 +90,7 @@ static void test_utf8_it(void)
     assert (i < 5);
     assert (string_get_cref(it) == tab[i]);
   }
+
   string_set_str (s, "H€llo René Chaînôr¬");
   assert(string_length_u(s) == 19);
   string_unicode_t tab2[19];