1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
https://bugs.gentoo.org/921490
https://www.sqlite.org/forum/forumpost/d16aeb397d
https://github.com/kovidgoyal/calibre/commit/0a23fabd5b4bff152047ba0cffbc2b67592645d1
https://github.com/kovidgoyal/calibre/commit/3349979551db05ec0c8754d7ea5a4e57a97f8a0a
https://github.com/kovidgoyal/calibre/commit/da82b673f09efbe9cdcc1e0900f0a9f04d8de052
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -209,10 +209,18 @@
rc = fts5CInstIterNext(&p->iter);
}
}
if( iPos==p->iRangeEnd ){
+ if( p->bOpen ){
+ if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
+ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
+ p->iOff = iEndOff;
+ }
+ fts5HighlightAppend(&rc, p, p->zClose, -1);
+ p->bOpen = 0;
+ }
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
}
return rc;
ADDED ext/fts5/test/fts5tokenizer2.test
Index: ext/fts5/test/fts5tokenizer2.test
==================================================================
--- /dev/null
+++ b/ext/fts5/test/fts5tokenizer2.test
@@ -0,0 +1,89 @@
+# 2023 Nov 03
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the built-in fts5 tokenizers.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
+proc get_tst_tokenizer {args} {
+ return "tst_tokenizer"
+}
+proc tst_tokenizer {flags txt} {
+ set token ""
+ set lTok [list]
+
+ foreach c [split $txt {}] {
+ if {$token==""} {
+ append token $c
+ } else {
+ set t1 [string is upper $token]
+ set t2 [string is upper $c]
+
+ if {$t1!=$t2} {
+ lappend lTok $token
+ set token ""
+ }
+ append token $c
+ }
+ }
+ if {$token!=""} { lappend lTok $token }
+
+ set iOff 0
+ foreach t $lTok {
+ set n [string length $t]
+ sqlite3_fts5_token $t $iOff [expr $iOff+$n]
+ incr iOff $n
+ }
+}
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
+}
+
+do_execsql_test 1.1 {
+ INSERT INTO t1 VALUES('AAdontBBmess');
+}
+
+do_execsql_test 1.2 {
+ SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.3 {
+ SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.4 {
+ SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
+} {>AA<dontBBmess}
+
+do_execsql_test 1.5 {
+ SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
+} {AA>dont<BBmess}
+
+do_execsql_test 1.6 {
+ SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
+} {AAdontBB>mess<}
+
+do_execsql_test 1.7 {
+ SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
+} {AAdont>BBmess<}
+
+
+finish_test
|
