Adding the sqlite sourcecode

ext/fts5/test/fts5cat.test

@@ -0,0 +1,76 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5cat
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
+  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid FROM t1('option');
+} {1}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
+
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co Mn'"
+  );
+  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
+
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    x, tokenize="unicode61 categories 'L* N* Co M*'"
+  );
+  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
+
+  INSERT INTO t2 VALUES ('สนามกีฬา');
+  INSERT INTO t3 VALUES ('สนามกีฬา');
+  INSERT INTO t4 VALUES ('สนามกีฬา');
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM t2t
+} {สนามก 1 1 ฬา 1 1}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t3t
+} {สนามกีฬา 1 1}
+
+do_execsql_test 1.5 {
+  SELECT * FROM t4t
+} {สนามกีฬา 1 1}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 "
+  CREATE VIRTUAL TABLE x1 USING fts5(c, 
+      tokenize=\"unicode61 categories ' \t'\");
+"
+
+do_catchsql_test 2.1 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tMYZ'\");
+" {1 {error in tokenizer constructor}}
+
+do_catchsql_test 2.2 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tXYZ'\");
+" {1 {error in tokenizer constructor}}
+
+
+finish_test