Enhance fts5txt2db.tcl, a script used to generate fts5/fts4 databases for performance testing.
FossilOrigin-Name: c646e40350e5aa91abcf52de61fb31275bad38f9
This commit is contained in:
dan
+145
-55
@@ -1,13 +1,127 @@
|
||||
|
||||
|
||||
proc usage {} {
|
||||
puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
|
||||
#-------------------------------------------------------------------------
|
||||
# Command line options processor.
|
||||
#
|
||||
proc command_line_error {O E {msg ""}} {
|
||||
if {$msg != ""} {
|
||||
puts stderr "Error: $msg"
|
||||
puts stderr ""
|
||||
}
|
||||
|
||||
set L [list]
|
||||
foreach o $O {
|
||||
if {[llength $o]==1} {
|
||||
lappend L [string toupper $o]
|
||||
}
|
||||
}
|
||||
|
||||
puts stderr "Usage: $::argv0 ?SWITCHES? $L"
|
||||
puts stderr ""
|
||||
puts stderr "Options are"
|
||||
puts stderr " -fts5"
|
||||
puts stderr " -fts4"
|
||||
puts stderr " -colsize <list of column sizes>"
|
||||
puts stderr {
|
||||
puts stderr "Switches are:"
|
||||
foreach o $O {
|
||||
if {[llength $o]==3} {
|
||||
foreach {a b c} $o {}
|
||||
puts stderr [format " -%-15s %s (default \"%s\")" "$a VAL" $c $b]
|
||||
} elseif {[llength $o]==2} {
|
||||
foreach {a b} $o {}
|
||||
puts stderr [format " -%-15s %s" $a $b]
|
||||
}
|
||||
}
|
||||
puts stderr ""
|
||||
puts stderr $E
|
||||
exit -1
|
||||
}
|
||||
|
||||
proc process_command_line {avar lArgs O E} {
|
||||
|
||||
upvar $avar A
|
||||
set zTrailing "" ;# True if ... is present in $O
|
||||
set lPosargs [list]
|
||||
|
||||
# Populate A() with default values. Also, for each switch in the command
|
||||
# line spec, set an entry in the idx() array as follows:
|
||||
#
|
||||
# {tblname t1 "table name to use"}
|
||||
# -> [set idx(-tblname) {tblname t1 "table name to use"}
|
||||
#
|
||||
# For each position parameter, append its name to $lPosargs. If the ...
|
||||
# specifier is present, set $zTrailing to the name of the prefix.
|
||||
#
|
||||
foreach o $O {
|
||||
set nm [lindex $o 0]
|
||||
set nArg [llength $o]
|
||||
switch -- $nArg {
|
||||
1 {
|
||||
if {[string range $nm end-2 end]=="..."} {
|
||||
set zTrailing [string range $nm 0 end-3]
|
||||
} else {
|
||||
lappend lPosargs $nm
|
||||
}
|
||||
}
|
||||
2 {
|
||||
set A($nm) 0
|
||||
set idx(-$nm) $o
|
||||
}
|
||||
3 {
|
||||
set A($nm) [lindex $o 1]
|
||||
set idx(-$nm) $o
|
||||
}
|
||||
default {
|
||||
error "Error in command line specification"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Set explicitly specified option values
|
||||
#
|
||||
set nArg [llength $lArgs]
|
||||
for {set i 0} {$i < $nArg} {incr i} {
|
||||
set opt [lindex $lArgs $i]
|
||||
if {[string range $opt 0 0]!="-" || $opt=="--"} break
|
||||
set c [array names idx "${opt}*"]
|
||||
if {[llength $c]==0} { command_line_error $O $E "Unrecognized option: $opt"}
|
||||
if {[llength $c]>1} { command_line_error $O $E "Ambiguous option: $opt"}
|
||||
|
||||
if {[llength $idx($c)]==3} {
|
||||
if {$i==[llength $lArgs]-1} {
|
||||
command_line_error $O $E "Option requires argument: $c"
|
||||
}
|
||||
incr i
|
||||
set A([lindex $idx($c) 0]) [lindex $lArgs $i]
|
||||
} else {
|
||||
set A([lindex $idx($c) 0]) 1
|
||||
}
|
||||
}
|
||||
|
||||
# Deal with position arguments.
|
||||
#
|
||||
set nPosarg [llength $lPosargs]
|
||||
set nRem [expr $nArg - $i]
|
||||
if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} {
|
||||
command_line_error $O $E
|
||||
}
|
||||
for {set j 0} {$j < $nPosarg} {incr j} {
|
||||
set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]]
|
||||
}
|
||||
if {$zTrailing!=""} {
|
||||
set A($zTrailing) [lrange $lArgs [expr $j+$i] end]
|
||||
}
|
||||
}
|
||||
# End of command line options processor.
|
||||
#-------------------------------------------------------------------------
|
||||
|
||||
|
||||
process_command_line A $argv {
|
||||
{fts5 "use fts5"}
|
||||
{fts4 "use fts4"}
|
||||
{colsize "10 10 10" "list of column sizes"}
|
||||
{tblname "t1" "table name to create"}
|
||||
{detail "full" "Fts5 detail mode to use"}
|
||||
{repeat 1 "Load each file this many times"}
|
||||
database
|
||||
file...
|
||||
} {
|
||||
This script is designed to create fts4/5 tables with more than one column.
|
||||
The -colsize option should be set to a Tcl list of integer values, one for
|
||||
each column in the table. Each value is the number of tokens that will be
|
||||
@@ -22,59 +136,27 @@ of the -colsize list. The next N2 are used for the second column of the first
|
||||
row, and so on. Rows are added to the table until the entire list of tokens
|
||||
is exhausted.
|
||||
}
|
||||
exit -1
|
||||
|
||||
if {$A(fts4)} {
|
||||
set A(fts) fts4
|
||||
} else {
|
||||
set A(fts) fts5
|
||||
}
|
||||
|
||||
set O(aColSize) [list 10 10 10]
|
||||
set O(tblname) t1
|
||||
set O(fts) fts5
|
||||
|
||||
|
||||
set options_with_values {-colsize}
|
||||
|
||||
for {set i 0} {$i < [llength $argv]} {incr i} {
|
||||
set opt [lindex $argv $i]
|
||||
if {[string range $opt 0 0]!="-"} break
|
||||
|
||||
if {[lsearch $options_with_values $opt]>=0} {
|
||||
incr i
|
||||
if {$i==[llength $argv]} usage
|
||||
set val [lindex $argv $i]
|
||||
}
|
||||
|
||||
switch -- $opt {
|
||||
-colsize {
|
||||
set O(aColSize) $val
|
||||
}
|
||||
|
||||
-fts4 {
|
||||
set O(fts) fts4
|
||||
}
|
||||
|
||||
-fts5 {
|
||||
set O(fts) fts5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if {$i > [llength $argv]-2} usage
|
||||
set O(db) [lindex $argv $i]
|
||||
set O(files) [lrange $argv [expr $i+1] end]
|
||||
|
||||
sqlite3 db $O(db)
|
||||
sqlite3 db $A(database)
|
||||
|
||||
# Create the FTS table in the db. Return a list of the table columns.
|
||||
#
|
||||
proc create_table {} {
|
||||
global O
|
||||
global A
|
||||
set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
|
||||
|
||||
set nCol [llength $O(aColSize)]
|
||||
set nCol [llength $A(colsize)]
|
||||
set cols [lrange $cols 0 [expr $nCol-1]]
|
||||
|
||||
set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
|
||||
set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) ("
|
||||
append sql [join $cols ,]
|
||||
append sql ");"
|
||||
if {$A(fts)=="fts5"} { append sql ",detail=$A(detail));" }
|
||||
|
||||
db eval $sql
|
||||
return $cols
|
||||
@@ -89,27 +171,35 @@ proc readfile {file} {
|
||||
split $data
|
||||
}
|
||||
|
||||
proc repeat {L n} {
|
||||
set res [list]
|
||||
for {set i 0} {$i < $n} {incr i} {
|
||||
set res [concat $res $L]
|
||||
}
|
||||
set res
|
||||
}
|
||||
|
||||
|
||||
# Load all the data into a big list of tokens.
|
||||
#
|
||||
set tokens [list]
|
||||
foreach f $O(files) {
|
||||
set tokens [concat $tokens [readfile $f]]
|
||||
foreach f $A(file) {
|
||||
set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]]
|
||||
}
|
||||
|
||||
set N [llength $tokens]
|
||||
set i 0
|
||||
set cols [create_table]
|
||||
set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
|
||||
set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
|
||||
foreach c [lrange $cols 1 end] {
|
||||
append sql ", \$A($c)"
|
||||
append sql ", \$R($c)"
|
||||
}
|
||||
append sql ")"
|
||||
|
||||
db eval BEGIN
|
||||
while {$i < $N} {
|
||||
foreach c $cols s $O(aColSize) {
|
||||
set A($c) [lrange $tokens $i [expr $i+$s-1]]
|
||||
foreach c $cols s $A(colsize) {
|
||||
set R($c) [lrange $tokens $i [expr $i+$s-1]]
|
||||
incr i $s
|
||||
}
|
||||
db eval $sql
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
C Fix\san\sfts5\sproblem\swith\susing\sboth\sxPhraseFirst()\sand\sxPhraseFirstColumn()\swithin\sa\ssingle\sstatement\sin\sdetail=col\smode.
|
||||
D 2016-01-23T18:51:59.865
|
||||
C Enhance\sfts5txt2db.tcl,\sa\sscript\sused\sto\sgenerate\sfts5/fts4\sdatabases\sfor\sperformance\stesting.
|
||||
D 2016-01-26T17:08:22.193
|
||||
F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
|
||||
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
|
||||
F Makefile.msc 1708a78eda223b6daa302b140037fcc214a779f9
|
||||
@@ -189,7 +189,7 @@ F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
|
||||
F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
|
||||
F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477
|
||||
F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221
|
||||
F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
|
||||
F ext/fts5/tool/fts5txt2db.tcl ae308338b2da1646dea456ab66706acdde8c714e
|
||||
F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
|
||||
F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
|
||||
F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
|
||||
@@ -1419,7 +1419,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
|
||||
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
|
||||
P ceccc9ad788fb4da9120915741995b9f088f85ff
|
||||
R e7d7753020b6f70b109a92a8ac037ba7
|
||||
P 72d53699bf0dcdb9d2a22e229989d7435f061399
|
||||
R e572f7330a1bdca73a606e620f2735b9
|
||||
U dan
|
||||
Z 648645363fb131dd10e5d7321a173b9d
|
||||
Z 9f71d918f9bd09bf4fa550347b1ed369
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
72d53699bf0dcdb9d2a22e229989d7435f061399
|
||||
c646e40350e5aa91abcf52de61fb31275bad38f9
|
||||
Reference in New Issue
Block a user