Fix some documentation issues in fts5.
FossilOrigin-Name: 512e1bdb4093b59d1494dfc63391476eadd52aea
This commit is contained in:
dan
+130
-35
@@ -17,14 +17,17 @@
|
||||
|
||||
set ::fts5_docs_output ""
|
||||
if {[info commands hd_putsnl]==""} {
|
||||
if {[llength $argv]>0} { set ::extract_api_docs_mode [lindex $argv 0] }
|
||||
proc output {text} {
|
||||
puts $text
|
||||
}
|
||||
} else {
|
||||
proc output {text} {
|
||||
append ::fts5_docs_output $text
|
||||
append ::fts5_docs_output "$text\n"
|
||||
}
|
||||
}
|
||||
if {[info exists ::extract_api_docs_mode]==0} {set ::extract_api_docs_mode api}
|
||||
|
||||
|
||||
set input_file [file join [file dir [info script]] fts5.h]
|
||||
set fd [open $input_file]
|
||||
@@ -41,7 +44,7 @@ close $fd
|
||||
proc get_struct_members {data} {
|
||||
|
||||
# Extract the structure definition from the fts5.h file.
|
||||
regexp "struct Fts5ExtensionApi {(.*)};" $data -> defn
|
||||
regexp "struct Fts5ExtensionApi {(.*?)};" $data -> defn
|
||||
|
||||
# Remove all comments from the structure definition
|
||||
regsub -all {/[*].*?[*]/} $defn {} defn2
|
||||
@@ -95,45 +98,137 @@ proc get_struct_docs {data names} {
|
||||
set res
|
||||
}
|
||||
|
||||
# Initialize global array M as a map from Fts5StructureApi member name
|
||||
# to member definition. i.e.
|
||||
#
|
||||
# iVersion -> {int iVersion}
|
||||
# xUserData -> {void *(*xUserData)(Fts5Context*)}
|
||||
# ...
|
||||
#
|
||||
array set M [get_struct_members $data]
|
||||
proc get_tokenizer_docs {data} {
|
||||
regexp {(xCreate:.*?)[*]/} $data -> docs
|
||||
|
||||
# Initialize global list D as a map from section name to documentation
|
||||
# text. Most (all?) section names are structure member names.
|
||||
#
|
||||
set D [get_struct_docs $data [array names M]]
|
||||
|
||||
foreach {hdr docs} $D {
|
||||
if {[info exists M($hdr)]} {
|
||||
set hdr $M($hdr)
|
||||
}
|
||||
output "<h style=\"font-size:1.4em;background-color:#EEEEEE;display:block\"><pre> $hdr</pre></h>"
|
||||
|
||||
set mode ""
|
||||
set bEmpty 1
|
||||
set res "<dl>\n"
|
||||
foreach line [split [string trim $docs] "\n"] {
|
||||
if {[string trim $line]==""} {
|
||||
if {$mode != ""} {output "</$mode>"}
|
||||
set mode ""
|
||||
} elseif {$mode == ""} {
|
||||
if {[regexp {^ } $line]} {
|
||||
set mode codeblock
|
||||
} else {
|
||||
set mode p
|
||||
}
|
||||
output "<$mode>"
|
||||
regexp {[*][*](.*)} $line -> line
|
||||
if {[regexp {^ ?x.*:} $line]} {
|
||||
append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
|
||||
continue
|
||||
}
|
||||
if {[string trim $line] == ""} {
|
||||
append res "<p>\n"
|
||||
} else {
|
||||
append res "$line\n"
|
||||
}
|
||||
output $line
|
||||
}
|
||||
if {$mode != ""} {output "</$mode>"}
|
||||
append res "</dl>\n"
|
||||
|
||||
set res
|
||||
}
|
||||
|
||||
proc get_api_docs {data} {
|
||||
# Initialize global array M as a map from Fts5StructureApi member name
|
||||
# to member definition. i.e.
|
||||
#
|
||||
# iVersion -> {int iVersion}
|
||||
# xUserData -> {void *(*xUserData)(Fts5Context*)}
|
||||
# ...
|
||||
#
|
||||
array set M [get_struct_members $data]
|
||||
|
||||
# Initialize global list D as a map from section name to documentation
|
||||
# text. Most (all?) section names are structure member names.
|
||||
#
|
||||
set D [get_struct_docs $data [array names M]]
|
||||
|
||||
foreach {sub docs} $D {
|
||||
if {[info exists M($sub)]} {
|
||||
set hdr $M($sub)
|
||||
set link " id=$sub"
|
||||
} else {
|
||||
set link ""
|
||||
}
|
||||
|
||||
output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
|
||||
set style "padding-left:6ex;font-size:1.4em;display:block"
|
||||
output "<h style=\"$style\"><pre>$hdr</pre></h>"
|
||||
|
||||
set mode ""
|
||||
set bEmpty 1
|
||||
foreach line [split [string trim $docs] "\n"] {
|
||||
if {[string trim $line]==""} {
|
||||
if {$mode != ""} {output "</$mode>"}
|
||||
set mode ""
|
||||
} elseif {$mode == ""} {
|
||||
if {[regexp {^ } $line]} {
|
||||
set mode codeblock
|
||||
} else {
|
||||
set mode p
|
||||
}
|
||||
output "<$mode>"
|
||||
}
|
||||
output $line
|
||||
}
|
||||
if {$mode != ""} {output "</$mode>"}
|
||||
}
|
||||
}
|
||||
|
||||
proc get_fts5_struct {data start end} {
|
||||
set res ""
|
||||
set bOut 0
|
||||
foreach line [split $data "\n"] {
|
||||
if {$bOut==0} {
|
||||
if {[regexp $start $line]} {
|
||||
set bOut 1
|
||||
}
|
||||
}
|
||||
|
||||
if {$bOut} {
|
||||
append res "$line\n"
|
||||
}
|
||||
|
||||
if {$bOut} {
|
||||
if {[regexp $end $line]} {
|
||||
set bOut 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
set map [list /* <i>/* */ */</i>]
|
||||
string map $map $res
|
||||
}
|
||||
|
||||
proc main {data} {
|
||||
switch $::extract_api_docs_mode {
|
||||
fts5_api {
|
||||
output [get_fts5_struct $data "typedef struct fts5_api" "^\};"]
|
||||
}
|
||||
|
||||
fts5_tokenizer {
|
||||
output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"]
|
||||
}
|
||||
|
||||
fts5_extension {
|
||||
output [get_fts5_struct $data "typedef.*Fts5ExtensionApi" "^.;"]
|
||||
}
|
||||
|
||||
Fts5ExtensionApi {
|
||||
set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
|
||||
set map [list]
|
||||
foreach {k v} [get_struct_members $data] {
|
||||
if {[string match x* $k]==0} continue
|
||||
lappend map $k "<a href=#$k>$k</a>"
|
||||
}
|
||||
output [string map $map $struct]
|
||||
}
|
||||
|
||||
api {
|
||||
get_api_docs $data
|
||||
}
|
||||
|
||||
tokenizer_api {
|
||||
output [get_tokenizer_docs $data]
|
||||
}
|
||||
|
||||
default {
|
||||
}
|
||||
}
|
||||
}
|
||||
main $data
|
||||
|
||||
set ::fts5_docs_output
|
||||
|
||||
|
||||
|
||||
+10
-12
@@ -100,7 +100,7 @@ typedef void (*fts5_extension_function)(
|
||||
** This API function is used to query the FTS table for phrase iPhrase
|
||||
** of the current query. Specifically, a query equivalent to:
|
||||
**
|
||||
** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
|
||||
** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid DESC
|
||||
**
|
||||
** with $p set to a phrase equivalent to the phrase iPhrase of the
|
||||
** current query is executed. For each row visited, the callback function
|
||||
@@ -212,10 +212,11 @@ struct Fts5ExtensionApi {
|
||||
**
|
||||
** The first argument passed to this function is a copy of the (void*)
|
||||
** pointer provided by the application when the fts5_tokenizer object
|
||||
** was registered with SQLite. The second and third arguments are an
|
||||
** array of nul-terminated strings containing the tokenizer arguments,
|
||||
** if any, specified as part of the CREATE VIRTUAL TABLE statement used
|
||||
** to create the fts5 table.
|
||||
** was registered with FTS5 (the third argument to xCreateTokenizer()).
|
||||
** The second and third arguments are an array of nul-terminated strings
|
||||
** containing the tokenizer arguments, if any, specified following the
|
||||
** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
|
||||
** to create the FTS5 table.
|
||||
**
|
||||
** The final argument is an output variable. If successful, (*ppOut)
|
||||
** should be set to point to the new tokenizer handle and SQLITE_OK
|
||||
@@ -240,12 +241,10 @@ struct Fts5ExtensionApi {
|
||||
** are a pointer to a buffer containing the token text, and the size of
|
||||
** the token in bytes. The 4th and 5th arguments are the byte offsets of
|
||||
** the first byte of and first byte immediately following the text from
|
||||
** which the token is derived within the input. The final argument is the
|
||||
** token position - the total number of tokens that appear before this one
|
||||
** in the input buffer.
|
||||
** which the token is derived within the input.
|
||||
**
|
||||
** The xToken() callback must be invoked with non-decreasing values of
|
||||
** the iPos parameter.
|
||||
** FTS5 assumes the xToken() callback is invoked for each token in the
|
||||
** order that they occur within the input text.
|
||||
**
|
||||
** If an xToken() callback returns any value other than SQLITE_OK, then
|
||||
** the tokenization should be abandoned and the xTokenize() method should
|
||||
@@ -256,9 +255,8 @@ struct Fts5ExtensionApi {
|
||||
** SQLITE_OK or SQLITE_DONE.
|
||||
**
|
||||
*/
|
||||
typedef struct fts5_tokenizer fts5_tokenizer;
|
||||
typedef struct Fts5Tokenizer Fts5Tokenizer;
|
||||
|
||||
typedef struct fts5_tokenizer fts5_tokenizer;
|
||||
struct fts5_tokenizer {
|
||||
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
|
||||
void (*xDelete)(Fts5Tokenizer*);
|
||||
|
||||
@@ -703,7 +703,7 @@ static int fts5ExprNearNextRowidMatch(
|
||||
*/
|
||||
static int fts5ExprNearNextMatch(
|
||||
Fts5Expr *pExpr, /* Expression that pNear is a part of */
|
||||
Fts5ExprNode *pNode,
|
||||
Fts5ExprNode *pNode, /* The "NEAR" node (FTS5_STRING) */
|
||||
int bFromValid,
|
||||
i64 iFrom
|
||||
){
|
||||
@@ -716,6 +716,9 @@ static int fts5ExprNearNextMatch(
|
||||
rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
|
||||
if( pNode->bEof || rc!=SQLITE_OK ) break;
|
||||
|
||||
/* Check that each phrase in the nearset matches the current row.
|
||||
** Populate the pPhrase->poslist buffers at the same time. If any
|
||||
** phrase is not a match, break out of the loop early. */
|
||||
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
|
||||
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
||||
if( pPhrase->nTerm>1 || pNear->iCol>=0 ){
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# 2014 Jan 08
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#***********************************************************************
|
||||
#
|
||||
# Tests focused on the NEAR operator.
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
set testprefix fts5near
|
||||
|
||||
proc do_near_test {tn doc near res} {
|
||||
uplevel [list do_execsql_test $tn "
|
||||
DELETE FROM t1;
|
||||
INSERT INTO t1 VALUES('$doc');
|
||||
SELECT count(*) FROM t1 WHERE t1 MATCH '$near';
|
||||
" $res]
|
||||
}
|
||||
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = 'simple tokenchars .')
|
||||
}
|
||||
|
||||
do_near_test 1.1 ". . a . . . b . ." { NEAR(a b, 5) } 1
|
||||
do_near_test 1.2 ". . a . . . b . ." { NEAR(a b, 4) } 1
|
||||
do_near_test 1.3 ". . a . . . b . ." { NEAR(a b, 3) } 1
|
||||
do_near_test 1.4 ". . a . . . b . ." { NEAR(a b, 2) } 0
|
||||
|
||||
do_near_test 1.5 ". . a . . . b . ." { NEAR(b a, 5) } 1
|
||||
do_near_test 1.6 ". . a . . . b . ." { NEAR(b a, 4) } 1
|
||||
do_near_test 1.7 ". . a . . . b . ." { NEAR(b a, 3) } 1
|
||||
do_near_test 1.8 ". . a . . . b . ." { NEAR(b a, 2) } 0
|
||||
|
||||
do_near_test 1.9 ". a b . . . c . ." { NEAR("a b" c, 3) } 1
|
||||
do_near_test 1.10 ". a b . . . c . ." { NEAR("a b" c, 2) } 0
|
||||
do_near_test 1.11 ". a b . . . c . ." { NEAR(c "a b", 3) } 1
|
||||
do_near_test 1.12 ". a b . . . c . ." { NEAR(c "a b", 2) } 0
|
||||
|
||||
do_near_test 1.13 ". a b . . . c d ." { NEAR(a+b c+d, 3) } 1
|
||||
do_near_test 1.14 ". a b . . . c d ." { NEAR(a+b c+d, 2) } 0
|
||||
do_near_test 1.15 ". a b . . . c d ." { NEAR(c+d a+b, 3) } 1
|
||||
do_near_test 1.16 ". a b . . . c d ." { NEAR(c+d a+b, 2) } 0
|
||||
|
||||
do_near_test 1.17 ". a b . . . c d ." { NEAR(a b c d, 5) } 1
|
||||
do_near_test 1.18 ". a b . . . c d ." { NEAR(a b c d, 4) } 0
|
||||
do_near_test 1.19 ". a b . . . c d ." { NEAR(a+b c d, 4) } 1
|
||||
|
||||
do_near_test 1.20 "a b c d e f g h i" { NEAR(b+c a+b+c+d i, 5) } 1
|
||||
do_near_test 1.21 "a b c d e f g h i" { NEAR(b+c a+b+c+d i, 4) } 0
|
||||
|
||||
do_near_test 1.22 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 5) } 1
|
||||
do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
|
||||
|
||||
do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
|
||||
do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
Reference in New Issue
Block a user