Compare commits

..

1 Commits

Author SHA1 Message Date
mistachkin 7547f78889 Expand scope of the SQLITE_DISABLE_MMAP define for the Win32 VFS.
FossilOrigin-Name: daa168f3dad995c4b7dd60e7717db90639e167c2
2013-04-11 22:52:44 +00:00
1300 changed files with 50490 additions and 317607 deletions
-1
View File
@@ -1 +0,0 @@
compat
-1
View File
@@ -1 +0,0 @@
compat/*
+138
View File
@@ -0,0 +1,138 @@
#!/usr/make
#
# Makefile for SQLITE
#
# This is a template makefile for SQLite. Most people prefer to
# use the autoconf generated "configure" script to generate the
# makefile automatically. But that does not work for everybody
# and in every situation. If you are having problems with the
# "configure" script, you might want to try this makefile as an
# alternative. Create a copy of this file, edit the parameters
# below and type "make".
#
#### The directory where to find the mingw32ce tools
MINGW32CE = /opt/mingw32ce/bin
#### The target prefix of the mingw32ce tools
TARGET = arm-wince-mingw32ce
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = ../sqlite
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DTHREADSAFE=1
#THREADSAFE = -DTHREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
#TLIBS = -lrt # fdatasync on Solaris 8
TLIBS =
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_WIN=1 -D_WIN32_WCE=1
#OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .exe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(MINGW32CE)/$(TARGET)-gcc -O2
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#### Tools used to build a static library.
#
#AR = ar cr
#AR = /opt/mingw/bin/i386-mingw32-ar cr
AR = $(MINGW32CE)/$(TARGET)-ar cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
RANLIB = $(MINGW32CE)/$(TARGET)-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(MINGW32CE)/$(TARGET)-gcc -shared
SO = dll
SHPREFIX =
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
# You should not have to change anything below this line
###############################################################################
include $(TOP)/main.mk
+68 -474
View File
@@ -15,23 +15,18 @@
# The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = @abs_srcdir@
TOP = @srcdir@
# C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = @BUILD_CC@ @BUILD_CFLAGS@
# TCC is the C Compile and options for use in building executables that
# C Compile and options for use in building executables that
# will run on the target platform. (BCC and TCC are usually the
# same unless your are cross-compiling.) Separate CC and CFLAGS macros
# are provide so that these aspects of the build process can be changed
# on the "make" command-line. Ex: "make CC=clang CFLAGS=-fsanitize=undefined"
# same unless your are cross-compiling.)
#
CC = @CC@
CFLAGS = @CPPFLAGS@ @CFLAGS@
TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
TCC = @CC@ @CPPFLAGS@ @CFLAGS@ -I. -I${TOP}/src -I${TOP}/ext/rtree
# Define this for the autoconf-based build, so that the code knows it can
# include the generated config.h
@@ -42,7 +37,7 @@ TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
# Omitting the define will cause extra debugging code to be inserted and
# includes extra comments when "EXPLAIN stmt" is used.
#
TCC += @TARGET_DEBUG@
TCC += @TARGET_DEBUG@ @XTHREADCONNECT@
# Compiler options needed for programs that use the TCL library.
#
@@ -55,7 +50,6 @@ LIBTCL = @TCL_LIB_SPEC@
# Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
# The library that programs using readline() must link against.
#
@@ -67,7 +61,7 @@ TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
# Any target libraries which libsqlite must be linked against
#
TLIBS = @LIBS@ $(LIBS)
TLIBS = @LIBS@
# Flags controlling use of the in memory btree implementation
#
@@ -89,9 +83,6 @@ TCC += $(OPT_FEATURE_FLAGS)
# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
TCC += $(OPTS)
# Add in compile-time options for some libraries used by extensions
TCC += @HAVE_ZLIB@
# Version numbers and release number for the SQLite being compiled.
#
VERSION = @VERSION@
@@ -160,6 +151,9 @@ LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS)
LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS)
LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL)
# nawk compatible awk.
NAWK = @AWK@
# You should not have to change anything below this line
###############################################################################
@@ -169,28 +163,23 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
#
LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
callback.lo complete.lo ctime.lo \
date.lo dbpage.lo dbstat.lo delete.lo \
callback.lo complete.lo ctime.lo date.lo delete.lo \
expr.lo fault.lo fkey.lo \
fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
fts3_tokenize_vtab.lo \
fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
fts5.lo \
func.lo global.lo hash.lo \
icu.lo insert.lo json1.lo legacy.lo loadext.lo \
icu.lo insert.lo journal.lo legacy.lo loadext.lo \
main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
memjournal.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo \
sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
table.lo tokenize.lo trigger.lo \
update.lo util.lo vacuum.lo \
vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
utf.lo vtab.lo
vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo
# Object files for the amalgamation.
#
@@ -219,8 +208,6 @@ SRC = \
$(TOP)/src/complete.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/dbpage.c \
$(TOP)/src/dbstat.c \
$(TOP)/src/delete.c \
$(TOP)/src/expr.c \
$(TOP)/src/fault.c \
@@ -231,6 +218,7 @@ SRC = \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/journal.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
@@ -241,7 +229,6 @@ SRC = \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
$(TOP)/src/msvc.h \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
@@ -251,10 +238,8 @@ SRC = \
$(TOP)/src/os.c \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_setup.h \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/os_win.h \
$(TOP)/src/pager.c \
$(TOP)/src/pager.h \
$(TOP)/src/parse.y \
@@ -262,7 +247,6 @@ SRC = \
$(TOP)/src/pcache.h \
$(TOP)/src/pcache1.c \
$(TOP)/src/pragma.c \
$(TOP)/src/pragma.h \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
@@ -270,16 +254,14 @@ SRC = \
$(TOP)/src/rowset.c \
$(TOP)/src/select.c \
$(TOP)/src/status.c \
$(TOP)/src/shell.c.in \
$(TOP)/src/shell.c \
$(TOP)/src/sqlite.h.in \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/threads.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/treeview.c \
$(TOP)/src/trigger.c \
$(TOP)/src/utf.c \
$(TOP)/src/update.c \
@@ -295,14 +277,10 @@ SRC = \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
$(TOP)/src/vxworks.h \
$(TOP)/src/wal.c \
$(TOP)/src/wal.h \
$(TOP)/src/walker.c \
$(TOP)/src/where.c \
$(TOP)/src/wherecode.c \
$(TOP)/src/whereexpr.c \
$(TOP)/src/whereInt.h
$(TOP)/src/where.c
# Source code for extensions
#
@@ -338,7 +316,6 @@ SRC += \
$(TOP)/ext/fts3/fts3_tokenizer.h \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_tokenizer1.c \
$(TOP)/ext/fts3/fts3_tokenize_vtab.c \
$(TOP)/ext/fts3/fts3_unicode.c \
$(TOP)/ext/fts3/fts3_unicode2.c \
$(TOP)/ext/fts3/fts3_write.c
@@ -348,15 +325,7 @@ SRC += \
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
SRC += \
$(TOP)/ext/session/sqlite3session.c \
$(TOP)/ext/session/sqlite3session.h
SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h \
$(TOP)/ext/rbu/sqlite3rbu.c
SRC += \
$(TOP)/ext/misc/json1.c \
$(TOP)/ext/misc/stmt.c
# Generated source code files
#
@@ -367,7 +336,6 @@ SRC += \
parse.c \
parse.h \
config.h \
shell.c \
sqlite3.h
# Source code to the test files.
@@ -385,70 +353,38 @@ TESTSRC = \
$(TOP)/src/test_autoext.c \
$(TOP)/src/test_async.c \
$(TOP)/src/test_backup.c \
$(TOP)/src/test_bestindex.c \
$(TOP)/src/test_blob.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_delete.c \
$(TOP)/src/test_demovfs.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_fs.c \
$(TOP)/src/test_func.c \
$(TOP)/src/test_fuzzer.c \
$(TOP)/src/test_hexio.c \
$(TOP)/src/test_init.c \
$(TOP)/src/test_intarray.c \
$(TOP)/src/test_journal.c \
$(TOP)/src/test_malloc.c \
$(TOP)/src/test_md5.c \
$(TOP)/src/test_multiplex.c \
$(TOP)/src/test_mutex.c \
$(TOP)/src/test_onefile.c \
$(TOP)/src/test_osinst.c \
$(TOP)/src/test_pcache.c \
$(TOP)/src/test_quota.c \
$(TOP)/src/test_regexp.c \
$(TOP)/src/test_rtree.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_superlock.c \
$(TOP)/src/test_syscall.c \
$(TOP)/src/test_tclsh.c \
$(TOP)/src/test_stat.c \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_windirent.c \
$(TOP)/src/test_wholenumber.c \
$(TOP)/src/test_wsd.c \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/ext/session/test_session.c \
$(TOP)/ext/rbu/test_rbu.c
# Statically linked extensions
#
TESTSRC += \
$(TOP)/ext/expert/sqlite3expert.c \
$(TOP)/ext/expert/test_expert.c \
$(TOP)/ext/misc/amatch.c \
$(TOP)/ext/misc/carray.c \
$(TOP)/ext/misc/closure.c \
$(TOP)/ext/misc/csv.c \
$(TOP)/ext/misc/eval.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/fuzzer.c \
$(TOP)/ext/fts5/fts5_tcl.c \
$(TOP)/ext/fts5/fts5_test_mi.c \
$(TOP)/ext/fts5/fts5_test_tok.c \
$(TOP)/ext/misc/ieee754.c \
$(TOP)/ext/misc/mmapwarm.c \
$(TOP)/ext/misc/nextchar.c \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/remember.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/spellfix.c \
$(TOP)/ext/misc/totype.c \
$(TOP)/ext/misc/unionvtab.c \
$(TOP)/ext/misc/wholenumber.c \
$(TOP)/ext/misc/zipfile.c
$(TOP)/ext/fts3/fts3_test.c
# Source code to the library files needed by the test fixture
#
@@ -460,13 +396,10 @@ TESTSRC2 = \
$(TOP)/src/build.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/dbpage.c \
$(TOP)/src/dbstat.c \
$(TOP)/src/expr.c \
$(TOP)/src/func.c \
$(TOP)/src/insert.c \
$(TOP)/src/wal.c \
$(TOP)/src/main.c \
$(TOP)/src/mem5.c \
$(TOP)/src/os.c \
$(TOP)/src/os_unix.c \
@@ -488,8 +421,6 @@ TESTSRC2 = \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/where.c \
$(TOP)/src/wherecode.c \
$(TOP)/src/whereexpr.c \
parse.c \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3_aux.c \
@@ -497,9 +428,7 @@ TESTSRC2 = \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_write.c \
$(TOP)/ext/async/sqlite3async.c \
$(TOP)/ext/session/sqlite3session.c \
$(TOP)/ext/misc/stmt.c
$(TOP)/ext/async/sqlite3async.c
# Header files used by all library source files.
#
@@ -509,25 +438,19 @@ HDR = \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
keywordhash.h \
$(TOP)/src/msvc.h \
$(TOP)/src/mutex.h \
opcodes.h \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_setup.h \
$(TOP)/src/os_win.h \
$(TOP)/src/pager.h \
$(TOP)/src/pcache.h \
parse.h \
$(TOP)/src/pragma.h \
sqlite3.h \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vxworks.h \
$(TOP)/src/whereInt.h \
config.h
# Header files used by extensions
@@ -552,47 +475,6 @@ EXTHDR += \
EXTHDR += \
$(TOP)/ext/rtree/sqlite3rtree.h
# executables needed for testing
#
TESTPROGS = \
testfixture$(TEXE) \
sqlite3$(TEXE) \
sqlite3_analyzer$(TEXE) \
sqldiff$(TEXE) \
dbhash$(TEXE) \
sqltclsh$(TEXE)
# Databases containing fuzzer test cases
#
FUZZDATA = \
$(TOP)/test/fuzzdata1.db \
$(TOP)/test/fuzzdata2.db \
$(TOP)/test/fuzzdata3.db \
$(TOP)/test/fuzzdata4.db \
$(TOP)/test/fuzzdata5.db
# Standard options to testfixture
#
TESTOPTS = --verbose=file --output=test-out.txt
# Extra compiler options for various shell tools
#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
#SHELL_OPT += -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT =
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
@@ -605,68 +487,25 @@ sqlite3.pc: $(TOP)/sqlite3.pc.in
./config.status
libsqlite3.la: $(LIBOBJ)
$(LTLINK) -no-undefined -o $@ $(LIBOBJ) $(TLIBS) \
$(LTLINK) -o $@ $(LIBOBJ) $(TLIBS) \
${ALLOWRELEASE} -rpath "$(libdir)" -version-info "8:6:8"
libtclsqlite3.la: tclsqlite.lo libsqlite3.la
$(LTLINK) -no-undefined -o $@ tclsqlite.lo \
$(LTLINK) -o $@ tclsqlite.lo \
libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \
-rpath "$(TCLLIBDIR)" \
-version-info "8:6:8" \
-avoid-version
sqlite3$(TEXE): shell.c sqlite3.c
$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
shell.c sqlite3.c \
sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h
$(LTLINK) $(READLINE_FLAGS) \
-o $@ $(TOP)/src/shell.c libsqlite3.la \
$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo
$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
srcck1$(BEXE): $(TOP)/tool/srcck1.c
$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
sourcetest: srcck1$(BEXE) sqlite3.c
./srcck1 sqlite3.c
fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
$(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c
$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c
$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
$(TLIBS) -rpath "$(libdir)"
MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20
mptest: mptester$(TEXE)
rm -f mptest.db
$(MPTEST1) --journalmode DELETE
$(MPTEST2) --journalmode WAL
$(MPTEST1) --journalmode WAL
$(MPTEST2) --journalmode PERSIST
$(MPTEST1) --journalmode PERSIST
$(MPTEST2) --journalmode TRUNCATE
$(MPTEST1) --journalmode TRUNCATE
$(MPTEST2) --journalmode DELETE
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
@@ -674,23 +513,18 @@ mptest: mptester$(TEXE)
# files are automatically generated. This target takes care of
# all that automatic generation.
#
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl $(OPTS) <tsrc/vdbe.c >vdbe.new
$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl <tsrc/vdbe.c >vdbe.new
mv vdbe.new tsrc/vdbe.c
cp fts5.c fts5.h tsrc
touch .target_source
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
cp tsrc/sqlite3ext.h .
cp $(TOP)/ext/session/sqlite3session.h .
sqlite3ext.h: .target_source
cp tsrc/sqlite3ext.h .
cp tsrc/shell.c tsrc/sqlite3ext.h .
tclsqlite3.c: sqlite3.c
echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
@@ -708,14 +542,9 @@ sqlite3.lo: sqlite3.c
# Rules to build the LEMON compiler generator
#
lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
$(BCC) -o $@ $(TOP)/tool/lemon.c
cp $(TOP)/tool/lempar.c .
# Rules to build the program that generates the source-id
#
mksourceid$(BEXE): $(TOP)/tool/mksourceid.c
$(BCC) -o $@ $(TOP)/tool/mksourceid.c
cp $(TOP)/src/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
@@ -770,12 +599,6 @@ ctime.lo: $(TOP)/src/ctime.c $(HDR)
date.lo: $(TOP)/src/date.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c
dbpage.lo: $(TOP)/src/dbpage.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
dbstat.lo: $(TOP)/src/dbstat.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c
delete.lo: $(TOP)/src/delete.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/delete.c
@@ -800,6 +623,9 @@ hash.lo: $(TOP)/src/hash.c $(HDR)
insert.lo: $(TOP)/src/insert.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c
journal.lo: $(TOP)/src/journal.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c
legacy.lo: $(TOP)/src/legacy.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c
@@ -890,15 +716,9 @@ status.lo: $(TOP)/src/status.c $(HDR)
table.lo: $(TOP)/src/table.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c
threads.lo: $(TOP)/src/threads.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c
tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c
treeview.lo: $(TOP)/src/treeview.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/treeview.c
trigger.lo: $(TOP)/src/trigger.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
@@ -947,17 +767,11 @@ walker.lo: $(TOP)/src/walker.c $(HDR)
where.lo: $(TOP)/src/where.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/where.c
wherecode.lo: $(TOP)/src/wherecode.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wherecode.c
whereexpr.lo: $(TOP)/src/whereexpr.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c
tclsqlite.lo: $(TOP)/src/tclsqlite.c $(HDR)
$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR)
$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR)
$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -968,47 +782,30 @@ tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl
$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
mv parse.h parse.h.temp
$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash$(BEXE) >keywordhash.h
# Source files that go into making shell.c
SHELL_SRC = \
$(TOP)/src/shell.c.in \
$(TOP)/ext/misc/appendvfs.c \
$(TOP)/ext/misc/shathree.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/completion.c \
$(TOP)/ext/misc/sqlar.c \
$(TOP)/ext/expert/sqlite3expert.c \
$(TOP)/ext/expert/sqlite3expert.h \
$(TOP)/ext/misc/zipfile.c \
$(TOP)/src/test_windirent.c
shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
# Rules to build the extension objects.
@@ -1061,9 +858,6 @@ fts3_tokenizer.lo: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
fts3_tokenizer1.lo: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
fts3_tokenize_vtab.lo: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c
fts3_unicode.lo: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c
@@ -1076,51 +870,6 @@ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
json1.lo: $(TOP)/ext/misc/json1.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
stmt.lo: $(TOP)/ext/misc/stmt.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
# FTS5 things
#
FTS5_SRC = \
$(TOP)/ext/fts5/fts5.h \
$(TOP)/ext/fts5/fts5Int.h \
$(TOP)/ext/fts5/fts5_aux.c \
$(TOP)/ext/fts5/fts5_buffer.c \
$(TOP)/ext/fts5/fts5_main.c \
$(TOP)/ext/fts5/fts5_config.c \
$(TOP)/ext/fts5/fts5_expr.c \
$(TOP)/ext/fts5/fts5_hash.c \
$(TOP)/ext/fts5/fts5_index.c \
fts5parse.c fts5parse.h \
$(TOP)/ext/fts5/fts5_storage.c \
$(TOP)/ext/fts5/fts5_tokenize.c \
$(TOP)/ext/fts5/fts5_unicode2.c \
$(TOP)/ext/fts5/fts5_varint.c \
$(TOP)/ext/fts5/fts5_vocab.c \
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
./lemon$(BEXE) $(OPTS) fts5parse.y
fts5parse.h: fts5parse.c
fts5.c: $(FTS5_SRC)
$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
fts5.lo: fts5.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c fts5.c
sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
# Rules to build the 'testfixture' application.
#
@@ -1129,14 +878,9 @@ sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
# necessary because the test fixture requires non-API symbols which are
# hidden when the library is built via the amalgamation).
#
TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
TESTFIXTURE_FLAGS += -DBUILD_sqlite
TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
TESTFIXTURE_SRC1 = sqlite3.c
@@ -1147,180 +891,39 @@ testfixture$(TEXE): $(TESTFIXTURE_SRC)
$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
# A very detailed test running most or all test cases
fulltest: $(TESTPROGS) fuzztest
./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
# Really really long testing
soaktest: $(TESTPROGS)
./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 $(TESTOPTS)
fulltest: testfixture$(TEXE) sqlite3$(TEXE)
./testfixture$(TEXE) $(TOP)/test/all.test
# Do extra testing but not everything.
fulltestonly: $(TESTPROGS) fuzztest
soaktest: testfixture$(TEXE) sqlite3$(TEXE)
./testfixture$(TEXE) $(TOP)/test/all.test -soak=1
fulltestonly: testfixture$(TEXE) sqlite3$(TEXE)
./testfixture$(TEXE) $(TOP)/test/full.test
# Fuzz testing
fuzztest: fuzzcheck$(TEXE) $(FUZZDATA)
./fuzzcheck$(TEXE) $(FUZZDATA)
test: testfixture$(TEXE) sqlite3$(TEXE)
./testfixture$(TEXE) $(TOP)/test/veryquick.test
fastfuzztest: fuzzcheck$(TEXE) $(FUZZDATA)
./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA)
valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
# The veryquick.test TCL tests.
#
tcltest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
# Minimal testing that runs in less than 3 minutes
#
quicktest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
test: fastfuzztest sourcetest $(TESTPROGS) tcltest
# Run a test using valgrind. This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest: $(TESTPROGS) valgrindfuzz
OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
# A very fast test that checks basic sanity. The name comes from
# the 60s-era electronics testing: "Turn it on and see if smoke
# comes out."
#
smoketest: $(TESTPROGS) fuzzcheck$(TEXE)
./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
echo "#define TCLSH 2" > $@
cat sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c >> $@
echo "static const char *tclsh_main_loop(void){" >> $@
echo "static const char *zMainloop = " >> $@
$(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@
echo "; return zMainloop; }" >> $@
sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
sqltclsh$(TEXE): sqltclsh.c
$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
$(LTLINK) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
CHECKER_DEPS =\
$(TOP)/tool/mkccode.tcl \
sqlite3.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/ext/repair/sqlite3_checker.tcl \
$(TOP)/ext/repair/checkindex.c \
$(TOP)/ext/repair/checkfreelist.c \
$(TOP)/ext/misc/btreeinfo.c \
$(TOP)/ext/repair/sqlite3_checker.c.in
sqlite3_checker.c: $(CHECKER_DEPS)
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
sqlite3_checker$(TEXE): sqlite3_checker.c
$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
$(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
showstat4$(TEXE): $(TOP)/tool/showstat4.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
showshm$(TEXE): $(TOP)/tool/showshm.c
$(LTLINK) -o $@ $(TOP)/tool/showshm.c
changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h
$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c
$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c
$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
# This target will fail if the SQLite amalgamation contains any exported
# symbols that do not begin with "sqlite3_". It is run as part of the
# releasetest.tcl script.
#
VALIDIDS=' sqlite3(changeset|changegroup|session)?_'
checksymbols: sqlite3.lo
nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0
echo '0 errors out of 1 tests'
# Build the amalgamation-autoconf package. The amalamgation-tarball target builds
# a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz.
# The snapshot-tarball target builds a tarball named by the SHA1 hash
#
amalgamation-tarball: sqlite3.c
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal
snapshot-tarball: sqlite3.c
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
#
THREADTEST3_SRC = $(TOP)/test/threadtest3.c \
$(TOP)/test/tt3_checkpoint.c \
$(TOP)/test/tt3_index.c \
$(TOP)/test/tt3_vacuum.c \
$(TOP)/test/tt3_stress.c \
$(TOP)/test/tt3_lookaside1.c
threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
$(LTLINK) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.lo -o $@ $(TLIBS)
threadtest: threadtest3$(TEXE)
./threadtest3$(TEXE)
releasetest:
$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
# Standard install and cleanup targets
#
lib_install: libsqlite3.la
$(INSTALL) -d $(DESTDIR)$(libdir)
$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
install: sqlite3$(BEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
$(INSTALL) -d $(DESTDIR)$(bindir)
$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
$(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir)
$(INSTALL) -d $(DESTDIR)$(includedir)
$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
@@ -1328,7 +931,7 @@ install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal
$(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir)
pkgIndex.tcl:
echo 'package ifneeded sqlite3 $(RELEASE) [list load $(TCLLIBDIR)/libtclsqlite3$(SHLIB_SUFFIX) sqlite3]' > $@
echo 'package ifneeded sqlite3 $(RELEASE) [list load $(TCLLIBDIR)/libtclsqlite3.so sqlite3]' > $@
tcl_install: lib_install libtclsqlite3.la pkgIndex.tcl
$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
@@ -1342,12 +945,10 @@ clean:
rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
rm -f mkkeywordhash$(BEXE) keywordhash.h
rm -f *.da *.bb *.bbg gmon.out
rm -rf quota2a quota2b quota2c
rm -rf tsrc .target_source
rm -f tclsqlite3$(TEXE)
rm -f testfixture$(TEXE) test.db
rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
rm -f wordcount$(TEXE) changeset$(TEXE)
rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
rm -f sqlite3.c
rm -f sqlite3rc.h
@@ -1355,16 +956,9 @@ clean:
rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
rm -f sqlite-*-output.vsix
rm -f mptester mptester.exe
rm -f rbu rbu.exe
rm -f srcck1 srcck1.exe
rm -f fuzzershell fuzzershell.exe
rm -f fuzzcheck fuzzcheck.exe
rm -f sqldiff sqldiff.exe
rm -f dbhash dbhash.exe
rm -f fts5.* fts5parse.*
distclean: clean
rm -f config.h config.log config.status libtool Makefile sqlite3.pc
rm -f config.log config.status libtool Makefile sqlite3.pc
#
# Windows section
+5
View File
@@ -118,6 +118,11 @@ READLINE_FLAGS =
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
# You should not have to change anything below this line
###############################################################################
include $(TOP)/main.mk
+359 -1555
View File
File diff suppressed because it is too large Load Diff
+667
View File
@@ -0,0 +1,667 @@
#!/usr/make
#
# Makefile for SQLITE on VxWorks
ifeq ($(FORCPU),)
FORCPU = SH32gnule
endif
TOOL_FAMILY = gnu
include $(WIND_USR)/tool/gnu/make.$(FORCPU)
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = .
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DSQLITE_THREADSAFE=1
#THREADSAFE = -DSQLITE_THREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
ifeq ($(CPU),SH32)
# for SH4 shared library
TLIBS_SHARED += -L$(WIND_USR)/lib/sh/SH32/commonle/PIC
else
# for all other CPUs shared library
TLIBS_SHARED += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
endif
# for static library
TLIBS += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_UNIX=1 $(THREADSAFE)
OPTS += -DSQLITE_OMIT_LOAD_EXTENSION=1
OPTS += -DSQLITE_ENABLE_LOCKING_STYLE=1
OPTS += -DSQLITE_THREAD_OVERRIDE_LOCK=0
OPTS += -DSQLITE_ENABLE_COLUMN_METADATA=1
OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .vxe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(CC) $(DEFINE_CC) -O2 -g -mrtp $(CC_ARCH_SPEC) -D_REENTRANT=1 -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
TCC += -I$(WIND_USR)/h -I$(WIND_USR)/h/wrn/coreip
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#TCC_SHARED = $(TCC) -fPIC
TCC_SHARED = $(TCC)
#### Tools used to build a static library.
#
#ARX = ar cr
#ARX = /opt/mingw/bin/i386-mingw32-ar cr
AR += cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(CC) $(DEFINE_CC) -mrtp -shared $(CC_ARCH_SPEC) -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
SO = so
SHPREFIX = lib
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
#### Pasted and adapted main.mk file
###############################################################################
# The following macros should be defined before this script is
# invoked:
#
# TOP The toplevel directory of the source tree. This is the
# directory that contains this "Makefile.in" and the
# "configure.in" script.
#
# BCC C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
# THREADLIB Specify any extra linker options needed to make the library
# thread safe
#
# OPTS Extra compiler command-line options.
#
# EXE The suffix to add to executable files. ".exe" for windows
# and "" for Unix.
#
# TCC C Compiler and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
# AR Tools used to build a static library.
# RANLIB
#
# TCL_FLAGS Extra compiler options needed for programs that use the
# TCL library.
#
# LIBTCL Linker options needed to link against the TCL library.
#
# READLINE_FLAGS Compiler options needed for programs that use the
# readline() library.
#
# LIBREADLINE Linker options needed by programs using readline() must
# link against.
#
# NAWK Nawk compatible awk program. Older (obsolete?) solaris
# systems need this to avoid using the original AT&T AWK.
#
# Once the macros above are defined, the rest of this make script will
# build the SQLite library and testing tools.
################################################################################
# This is how we compile
#
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX_SHARED = $(TCC_SHARED) $(OPTS) -I. -I$(TOP)/src -I$(TOP) \
-I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 \
-I$(TOP)/ext/async
# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o \
backup.o bitvec.o btmutex.o btree.o build.o \
callback.o complete.o date.o delete.o expr.o fault.o \
fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
fts3_tokenizer.o fts3_tokenizer1.o \
func.o global.o hash.o \
icu.o insert.o journal.o legacy.o loadext.o \
main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
memjournal.o \
mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
notify.o opcodes.o os.o os_unix.o os_win.o \
pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
update.o util.o vacuum.o \
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \
walker.o where.o utf.o vtab.o
# All of the source code files.
#
SRC = \
$(TOP)/src/alter.c \
$(TOP)/src/analyze.c \
$(TOP)/src/attach.c \
$(TOP)/src/auth.c \
$(TOP)/src/backup.c \
$(TOP)/src/bitvec.c \
$(TOP)/src/btmutex.c \
$(TOP)/src/btree.c \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/build.c \
$(TOP)/src/callback.c \
$(TOP)/src/complete.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/delete.c \
$(TOP)/src/expr.c \
$(TOP)/src/fault.c \
$(TOP)/src/func.c \
$(TOP)/src/global.c \
$(TOP)/src/hash.c \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/journal.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
$(TOP)/src/malloc.c \
$(TOP)/src/mem0.c \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
$(TOP)/src/mutex_unix.c \
$(TOP)/src/mutex_w32.c \
$(TOP)/src/notify.c \
$(TOP)/src/os.c \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
$(TOP)/src/pager.h \
$(TOP)/src/parse.y \
$(TOP)/src/pcache.c \
$(TOP)/src/pcache.h \
$(TOP)/src/pcache1.c \
$(TOP)/src/pragma.c \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
$(TOP)/src/resolve.c \
$(TOP)/src/rowset.c \
$(TOP)/src/select.c \
$(TOP)/src/status.c \
$(TOP)/src/shell.c \
$(TOP)/src/sqlite.h.in \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/trigger.c \
$(TOP)/src/utf.c \
$(TOP)/src/update.c \
$(TOP)/src/util.c \
$(TOP)/src/vacuum.c \
$(TOP)/src/vdbe.c \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeapi.c \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
$(TOP)/src/walker.c \
$(TOP)/src/where.c
# Source code for extensions
#
SRC += \
$(TOP)/ext/fts1/fts1.c \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.c \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_porter.c \
$(TOP)/ext/fts1/fts1_tokenizer.h \
$(TOP)/ext/fts1/fts1_tokenizer1.c
SRC += \
$(TOP)/ext/fts2/fts2.c \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.c \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_icu.c \
$(TOP)/ext/fts2/fts2_porter.c \
$(TOP)/ext/fts2/fts2_tokenizer.h \
$(TOP)/ext/fts2/fts2_tokenizer.c \
$(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.c \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_icu.c \
$(TOP)/ext/fts3/fts3_porter.c \
$(TOP)/ext/fts3/fts3_tokenizer.h \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_tokenizer1.c
SRC += \
$(TOP)/ext/icu/sqliteicu.h \
$(TOP)/ext/icu/icu.c
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
# Generated source code files
#
SRC += \
keywordhash.h \
opcodes.c \
opcodes.h \
parse.c \
parse.h \
sqlite3.h
# Source code to the test files.
#
TESTSRC = \
$(TOP)/src/test1.c \
$(TOP)/src/test2.c \
$(TOP)/src/test3.c \
$(TOP)/src/test4.c \
$(TOP)/src/test5.c \
$(TOP)/src/test6.c \
$(TOP)/src/test7.c \
$(TOP)/src/test8.c \
$(TOP)/src/test9.c \
$(TOP)/src/test_autoext.c \
$(TOP)/src/test_async.c \
$(TOP)/src/test_backup.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_func.c \
$(TOP)/src/test_hexio.c \
$(TOP)/src/test_journal.c \
$(TOP)/src/test_malloc.c \
$(TOP)/src/test_md5.c \
$(TOP)/src/test_mutex.c \
$(TOP)/src/test_onefile.c \
$(TOP)/src/test_osinst.c \
$(TOP)/src/test_pcache.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_wsd.c \
#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
TESTSRC2 = \
$(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \
$(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c \
$(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \
$(TOP)/src/os_unix.c $(TOP)/src/os_win.c \
$(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \
$(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \
$(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c \
$(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c \
$(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/async/sqlite3async.c
# Header files used by all library source files.
#
HDR = \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
keywordhash.h \
$(TOP)/src/mutex.h \
opcodes.h \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/pager.h \
$(TOP)/src/pcache.h \
parse.h \
sqlite3.h \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeInt.h
# Header files used by extensions
#
EXTHDR += \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_tokenizer.h
EXTHDR += \
$(TOP)/ext/rtree/rtree.h
EXTHDR += \
$(TOP)/ext/icu/sqliteicu.h
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.a sqlite3$(EXE)
libsqlite3.a: $(LIBOBJ)
$(AR) libsqlite3.a $(LIBOBJ)
$(RANLIB) libsqlite3.a
$(SHPREFIX)sqlite3.$(SO): $(LIBOBJ)
$(MKSHLIB) -o $(SHPREFIX)sqlite3.$(SO) $(LIBOBJ) $(TLIBS_SHARED)
sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \
$(TOP)/src/shell.c \
$(LIBREADLINE) $(TLIBS) $(THREADLIB) -L. -lsqlite3
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
# build on the target system. Some of the C source code and header
# files are automatically generated. This target takes care of
# all that automatic generation.
#
target_source: $(SRC)
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
touch target_source
sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl
tclsh $(TOP)/tool/mksqlite3c.tcl
cp sqlite3.c tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl
tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl
tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
# Rules to build the LEMON compiler generator
#
lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
$(BCC) -o lemon $(TOP)/tool/lemon.c
cp $(TOP)/src/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
#
# parse.o
# opcodes.o
#
%.o: %.c $(HDR)
$(TCCX_SHARED) -c $<
# Rules to build individual *.o files from files in the src directory.
#
%.o: $(TOP)/src/%.c $(HDR)
$(TCCX_SHARED) -c $<
tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR)
$(TCCX_SHARED) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | \
$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon $(OPTS) parse.y
mv parse.h parse.h.temp
awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in
sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
-e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
$(TOP)/src/sqlite.h.in >sqlite3.h
keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash >keywordhash.h
# Rules to build the extension objects.
#
icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
fts2.o: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
fts2_hash.o: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
fts2_icu.o: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
fts2_porter.o: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
fts2_tokenizer.o: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
# Rules for building test programs and for running tests
#
tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a
$(TCCX_SHARED) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
# Rules to build the 'testfixture' application.
#
TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DSQLITE_ENABLE_FTS3=1 \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fulltest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test
soaktest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test -soak=1
fulltestonly: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/full.test
test: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/veryquick.test
sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
$(TOP)/tool/spaceanal.tcl
sed \
-e '/^#/d' \
-e 's,\\,\\\\,g' \
-e 's,",\\",g' \
-e 's,^,",' \
-e 's,$$,\\n",' \
$(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o sqlite3_analyzer$(EXE) \
$(LIBTCL) $(THREADLIB)
TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
./testfixture$(EXE) $(TOP)/test/loadext.test
clean:
rm -f *.o sqlite3$(EXE) libsqlite3.a sqlite3.h opcodes.*
rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
rm -f $(PUBLISH)
rm -f *.da *.bb *.bbg gmon.out
rm -rf quota2a quota2b quota2c
rm -rf tsrc target_source
rm -f testloadext.dll libtestloadext.so
rm -f sqlite3.c fts?amal.c tclsqlite3.c
rm -f sqlite3rc.h
rm -f shell.c sqlite3ext.h
rm -f $(SHPREFIX)sqlite3.$(SO)
+39
View File
@@ -0,0 +1,39 @@
This directory contains source code to
SQLite: An Embeddable SQL Database Engine
To compile the project, first create a directory in which to place
the build products. It is recommended, but not required, that the
build directory be separate from the source directory. Cd into the
build directory and then from the build directory run the configure
script found at the root of the source tree. Then run "make".
For example:
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make ;# Run the makefile.
make install ;# (Optional) Install the build products
The configure script uses autoconf 2.61 and libtool. If the configure
script does not work out for you, there is a generic makefile named
"Makefile.linux-gcc" in the top directory of the source tree that you
can copy and edit to suit your needs. Comments on the generic makefile
show what changes are needed.
The linux binaries on the website are created using the generic makefile,
not the configure script. The windows binaries on the website are created
using MinGW32 configured as a cross-compiler running under Linux. For
details, see the ./publish.sh script at the top-level of the source tree.
The developers do not use teh configure script.
SQLite does not require TCL to run, but a TCL installation is required
by the makefiles. SQLite contains a lot of generated code and TCL is
used to do much of that code generation. The makefile also requires
AWK.
Contacts:
http://www.sqlite.org/
-299
View File
@@ -1,299 +0,0 @@
<h1 align="center">SQLite Source Repository</h1>
This repository contains the complete source code for the SQLite database
engine. Some test scripts are also included. However, many other test scripts
and most of the documentation are managed separately.
If you are reading this on a Git mirror someplace, you are doing it wrong.
The [official repository](https://www.sqlite.org/src/) is better. Go there
now.
## Obtaining The Code
SQLite sources are managed using the
[Fossil](https://www.fossil-scm.org/), a distributed version control system
that was specifically designed to support SQLite development.
If you do not want to use Fossil, you can download tarballs or ZIP
archives as follows:
* Lastest trunk check-in:
<https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
<https://www.sqlite.org/src/zip/sqlite.zip>.
* Latest release:
<https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
<https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
* For other check-ins, substitute an appropriate branch name or
tag or hash prefix for "release" in the URLs of the previous
bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline)
to locate the check-in desired, click on its information page link,
then click on the "Tarball" or "ZIP Archive" links on the information
page.
If you do want to use Fossil to check out the source tree,
first install Fossil version 2.0 or later.
(Source tarballs and precompiled binaries available
[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is
a stand-alone program. To install, simply download or build the single
executable file and put that file someplace on your $PATH.)
Then run commands like this:
mkdir ~/sqlite
cd ~/sqlite
fossil clone https://www.sqlite.org/src sqlite.fossil
fossil open sqlite.fossil
After setting up a repository using the steps above, you can always
update to the lastest version using:
fossil update trunk ;# latest trunk check-in
fossil update release ;# latest official release
Or type "fossil ui" to get a web-based user interface.
## Compiling
First create a directory in which to place
the build products. It is recommended, but not required, that the
build directory be separate from the source directory. Cd into the
build directory and then from the build directory run the configure
script found at the root of the source tree. Then run "make".
For example:
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make ;# Run the makefile.
make sqlite3.c ;# Build the "amalgamation" source file
make test ;# Run some tests (requires Tcl)
See the makefile for additional targets.
The configure script uses autoconf 2.61 and libtool. If the configure
script does not work out for you, there is a generic makefile named
"Makefile.linux-gcc" in the top directory of the source tree that you
can copy and edit to suit your needs. Comments on the generic makefile
show what changes are needed.
## Using MSVC
On Windows, all applicable build products can be compiled with MSVC.
First open the command prompt window associated with the desired compiler
version (e.g. "Developer Command Prompt for VS2013"). Next, use NMAKE
with the provided "Makefile.msc" to build one of the supported targets.
For example:
mkdir bld
cd bld
nmake /f Makefile.msc TOP=..\sqlite
nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
nmake /f Makefile.msc test TOP=..\sqlite
There are several build options that can be set via the NMAKE command
line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument
to the "sqlite3.dll" command line above. When debugging into the SQLite
code, adding the "DEBUG=1" argument to one of the above command lines is
recommended.
SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
is required by the makefiles (including those for MSVC). SQLite contains
a lot of generated code and Tcl is used to do much of that code generation.
## Source Code Tour
Most of the core source files are in the **src/** subdirectory. The
**src/** folder also contains files used to build the "testfixture" test
harness. The names of the source files used by "testfixture" all begin
with "test".
The **src/** also contains the "shell.c" file
which is the main program for the "sqlite3.exe"
[command-line shell](https://sqlite.org/cli.html) and
the "tclsqlite.c" file which implements the
[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
(Historical note: SQLite began as a Tcl
extension and only later escaped to the wild as an independent library.)
Test scripts and programs are found in the **test/** subdirectory.
Addtional test code is found in other source repositories.
See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
additional information.
The **ext/** subdirectory contains code for extensions. The
Full-text search engine is in **ext/fts3**. The R-Tree engine is in
**ext/rtree**. The **ext/misc** subdirectory contains a number of
smaller, single-file extensions, such as a REGEXP operator.
The **tool/** subdirectory contains various scripts and programs used
for building generated source code files or for testing or for generating
accessory programs such as "sqlite3_analyzer(.exe)".
### Generated Source Code Files
Several of the C-language source files used by SQLite are generated from
other sources rather than being typed in manually by a programmer. This
section will summarize those automatically-generated files. To create all
of the automatically-generated files, simply run "make target&#95;source".
The "target&#95;source" make target will create a subdirectory "tsrc/" and
fill it with all the source files needed to build SQLite, both
manually-edited files and automatically-generated files.
The SQLite interface is defined by the **sqlite3.h** header file, which is
generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The
[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
The manifest.uuid file contains the SHA3 hash of the particular check-in
and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file
contains the current SQLite version number. The sqlite3.h header is really
just a copy of src/sqlite.h.in with the source-id and version number inserted
at just the right spots. Note that comment text in the sqlite3.h file is
used to generate much of the SQLite API documentation. The Tcl scripts
used to generate that documentation are in a separate source repository.
The SQL language parser is **parse.c** which is generate from a grammar in
the src/parse.y file. The conversion of "parse.y" into "parse.c" is done
by the [lemon](./doc/lemon.html) LALR(1) parser generator. The source code
for lemon is at tool/lemon.c. Lemon uses the tool/lempar.c file as a
template for generating its parser.
Lemon also generates the **parse.h** header file, at the same time it
generates parse.c. But the parse.h header file is
modified further (to add additional symbols) using the ./addopcodes.tcl
Tcl script.
The **opcodes.h** header file contains macros that define the numbers
corresponding to opcodes in the "VDBE" virtual machine. The opcodes.h
file is generated by the scanning the src/vdbe.c source file. The
Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
the **opcodes.c** source file, which contains a reverse mapping from
opcode-number to opcode-name that is used for EXPLAIN output.
The **keywordhash.h** header file contains the definition of a hash table
that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
the numeric codes used by the parse.c parser. The keywordhash.h file is
generated by a C-language program at tool mkkeywordhash.c.
The **pragma.h** header file contains various definitions used to parse
and implement the PRAGMA statements. The header is generated by a
script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
the **tool/mkpragmatab.tcl** file to insert the information needed by the
parser for your new PRAGMA, then run the script to regenerate the
**pragma.h** header file.
### The Amalgamation
All of the individual C source code and header files (both manually-edited
and automatically-generated) can be combined into a single big source file
**sqlite3.c** called "the amalgamation". The amalgamation is the recommended
way of using SQLite in a larger application. Combining all individual
source code files into a single big source code file allows the C compiler
to perform more cross-procedure analysis and generate better code. SQLite
runs about 5% faster when compiled from the amalgamation versus when compiled
from individual source files.
The amalgamation is generated from the tool/mksqlite3c.tcl Tcl script.
First, all of the individual source files must be gathered into the tsrc/
subdirectory (using the equivalent of "make target_source") then the
tool/mksqlite3c.tcl script is run to copy them all together in just the
right order while resolving internal "#include" references.
The amalgamation source file is more than 200K lines long. Some symbolic
debuggers (most notably MSVC) are unable to deal with files longer than 64K
lines. To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
can be run on the amalgamation to break it up into a single small C file
called **sqlite3-all.c** that does #include on about seven other files
named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**. In this way,
all of the source code is contained within a single translation unit so
that the compiler can do extra cross-procedure optimization, but no
individual source file exceeds 32K lines in length.
## How It All Fits Together
SQLite is modular in design.
See the [architectural description](http://www.sqlite.org/arch.html)
for details. Other documents that are useful in
(helping to understand how SQLite works include the
[file format](http://www.sqlite.org/fileformat2.html) description,
the [virtual machine](http://www.sqlite.org/opcode.html) that runs
prepared statements, the description of
[how transactions work](http://www.sqlite.org/atomiccommit.html), and
the [overview of the query planner](http://www.sqlite.org/optoverview.html).
Years of effort have gone into optimizating SQLite, both
for small size and high performance. And optimizations tend to result in
complex code. So there is a lot of complexity in the current SQLite
implementation. It will not be the easiest library in the world to hack.
Key files:
* **sqlite.h.in** - This file defines the public interface to the SQLite
library. Readers will need to be familiar with this interface before
trying to understand how the library works internally.
* **sqliteInt.h** - this header file defines many of the data objects
used internally by SQLite. In addition to "sqliteInt.h", some
subsystems have their own header files.
* **parse.y** - This file describes the LALR(1) grammar that SQLite uses
to parse SQL statements, and the actions that are taken at each step
in the parsing process.
* **vdbe.c** - This file implements the virtual machine that runs
prepared statements. There are various helper files whose names
begin with "vdbe". The VDBE has access to the vdbeInt.h header file
which defines internal data objects. The rest of SQLite interacts
with the VDBE through an interface defined by vdbe.h.
* **where.c** - This file (together with its helper files named
by "where*.c") analyzes the WHERE clause and generates
virtual machine code to run queries efficiently. This file is
sometimes called the "query optimizer". It has its own private
header file, whereInt.h, that defines data objects used internally.
* **btree.c** - This file contains the implementation of the B-Tree
storage engine used by SQLite. The interface to the rest of the system
is defined by "btree.h". The "btreeInt.h" header defines objects
used internally by btree.c and not published to the rest of the system.
* **pager.c** - This file contains the "pager" implementation, the
module that implements transactions. The "pager.h" header file
defines the interface between pager.c and the rest of the system.
* **os_unix.c** and **os_win.c** - These two files implement the interface
between SQLite and the underlying operating system using the run-time
pluggable VFS interface.
* **shell.c.in** - This file is not part of the core SQLite library. This
is the file that, when linked against sqlite3.a, generates the
"sqlite3.exe" command-line shell. The "shell.c.in" file is transformed
into "shell.c" as part of the build process.
* **tclsqlite.c** - This file implements the Tcl bindings for SQLite. It
is not part of the core SQLite library. But as most of the tests in this
repository are written in Tcl, the Tcl language bindings are important.
* **test*.c** - Files in the src/ folder that begin with "test" go into
building the "testfixture.exe" program. The testfixture.exe program is
an enhanced Tcl shell. The testfixture.exe program runs scripts in the
test/ folder to validate the core SQLite code. The testfixture program
(and some other test programs too) is build and run when you type
"make test".
* **ext/misc/json1.c** - This file implements the various JSON functions
that are build into SQLite.
There are many other source files. Each has a succinct header comment that
describes its purpose and role within the larger system.
## Contacts
The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
with geographically distributed backups at
[http://www2.sqlite.org/](http://www2.sqlite.org) and
[http://www3.sqlite.org/](http://www3.sqlite.org).
+1 -1
View File
@@ -1 +1 @@
3.22.0
3.7.17
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/awk
#
# This script appends additional token codes to the end of the
# parse.h file that lemon generates. These extra token codes are
# not used by the parser. But they are used by the tokenizer and/or
# the code generator.
#
#
BEGIN {
max = 0
}
/^#define TK_/ {
print $0
if( max<$3 ) max = $3
}
END {
printf "#define TK_%-29s %4d\n", "TO_TEXT", ++max
printf "#define TK_%-29s %4d\n", "TO_BLOB", ++max
printf "#define TK_%-29s %4d\n", "TO_NUMERIC", ++max
printf "#define TK_%-29s %4d\n", "TO_INT", ++max
printf "#define TK_%-29s %4d\n", "TO_REAL", ++max
printf "#define TK_%-29s %4d\n", "ISNOT", ++max
printf "#define TK_%-29s %4d\n", "END_OF_FILE", ++max
printf "#define TK_%-29s %4d\n", "ILLEGAL", ++max
printf "#define TK_%-29s %4d\n", "SPACE", ++max
printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", ++max
printf "#define TK_%-29s %4d\n", "FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "COLUMN", ++max
printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max
printf "#define TK_%-29s %4d\n", "CONST_FUNC", ++max
printf "#define TK_%-29s %4d\n", "UMINUS", ++max
printf "#define TK_%-29s %4d\n", "UPLUS", ++max
}
-370
View File
@@ -1,370 +0,0 @@
Installation Instructions
*************************
Copyright (C) 1994-1996, 1999-2002, 2004-2011 Free Software Foundation,
Inc.
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
notice and this notice are preserved. This file is offered as-is,
without warranty of any kind.
Basic Installation
==================
Briefly, the shell commands `./configure; make; make install' should
configure, build, and install this package. The following
more-detailed instructions are generic; see the `README' file for
instructions specific to this package. Some packages provide this
`INSTALL' file but do not implement all of the features documented
below. The lack of an optional feature in a given package is not
necessarily a bug. More recommendations for GNU packages can be found
in *note Makefile Conventions: (standards)Makefile Conventions.
The `configure' shell script attempts to guess correct values for
various system-dependent variables used during compilation. It uses
those values to create a `Makefile' in each directory of the package.
It may also create one or more `.h' files containing system-dependent
definitions. Finally, it creates a shell script `config.status' that
you can run in the future to recreate the current configuration, and a
file `config.log' containing compiler output (useful mainly for
debugging `configure').
It can also use an optional file (typically called `config.cache'
and enabled with `--cache-file=config.cache' or simply `-C') that saves
the results of its tests to speed up reconfiguring. Caching is
disabled by default to prevent problems with accidental use of stale
cache files.
If you need to do unusual things to compile the package, please try
to figure out how `configure' could check whether to do them, and mail
diffs or instructions to the address given in the `README' so they can
be considered for the next release. If you are using the cache, and at
some point `config.cache' contains results you don't want to keep, you
may remove or edit it.
The file `configure.ac' (or `configure.in') is used to create
`configure' by a program called `autoconf'. You need `configure.ac' if
you want to change it or regenerate `configure' using a newer version
of `autoconf'.
The simplest way to compile this package is:
1. `cd' to the directory containing the package's source code and type
`./configure' to configure the package for your system.
Running `configure' might take a while. While running, it prints
some messages telling which features it is checking for.
2. Type `make' to compile the package.
3. Optionally, type `make check' to run any self-tests that come with
the package, generally using the just-built uninstalled binaries.
4. Type `make install' to install the programs and any data files and
documentation. When installing into a prefix owned by root, it is
recommended that the package be configured and built as a regular
user, and only the `make install' phase executed with root
privileges.
5. Optionally, type `make installcheck' to repeat any self-tests, but
this time using the binaries in their final installed location.
This target does not install anything. Running this target as a
regular user, particularly if the prior `make install' required
root privileges, verifies that the installation completed
correctly.
6. You can remove the program binaries and object files from the
source code directory by typing `make clean'. To also remove the
files that `configure' created (so you can compile the package for
a different kind of computer), type `make distclean'. There is
also a `make maintainer-clean' target, but that is intended mainly
for the package's developers. If you use it, you may have to get
all sorts of other programs in order to regenerate files that came
with the distribution.
7. Often, you can also type `make uninstall' to remove the installed
files again. In practice, not all packages have tested that
uninstallation works correctly, even though it is required by the
GNU Coding Standards.
8. Some packages, particularly those that use Automake, provide `make
distcheck', which can by used by developers to test that all other
targets like `make install' and `make uninstall' work correctly.
This target is generally not run by end users.
Compilers and Options
=====================
Some systems require unusual options for compilation or linking that
the `configure' script does not know about. Run `./configure --help'
for details on some of the pertinent environment variables.
You can give `configure' initial values for configuration parameters
by setting variables in the command line or in the environment. Here
is an example:
./configure CC=c99 CFLAGS=-g LIBS=-lposix
*Note Defining Variables::, for more details.
Compiling For Multiple Architectures
====================================
You can compile the package for more than one kind of computer at the
same time, by placing the object files for each architecture in their
own directory. To do this, you can use GNU `make'. `cd' to the
directory where you want the object files and executables to go and run
the `configure' script. `configure' automatically checks for the
source code in the directory that `configure' is in and in `..'. This
is known as a "VPATH" build.
With a non-GNU `make', it is safer to compile the package for one
architecture at a time in the source code directory. After you have
installed the package for one architecture, use `make distclean' before
reconfiguring for another architecture.
On MacOS X 10.5 and later systems, you can create libraries and
executables that work on multiple system types--known as "fat" or
"universal" binaries--by specifying multiple `-arch' options to the
compiler but only a single `-arch' option to the preprocessor. Like
this:
./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \
CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \
CPP="gcc -E" CXXCPP="g++ -E"
This is not guaranteed to produce working output in all cases, you
may have to build one architecture at a time and combine the results
using the `lipo' tool if you have problems.
Installation Names
==================
By default, `make install' installs the package's commands under
`/usr/local/bin', include files under `/usr/local/include', etc. You
can specify an installation prefix other than `/usr/local' by giving
`configure' the option `--prefix=PREFIX', where PREFIX must be an
absolute file name.
You can specify separate installation prefixes for
architecture-specific files and architecture-independent files. If you
pass the option `--exec-prefix=PREFIX' to `configure', the package uses
PREFIX as the prefix for installing programs and libraries.
Documentation and other data files still use the regular prefix.
In addition, if you use an unusual directory layout you can give
options like `--bindir=DIR' to specify different values for particular
kinds of files. Run `configure --help' for a list of the directories
you can set and what kinds of files go in them. In general, the
default for these options is expressed in terms of `${prefix}', so that
specifying just `--prefix' will affect all of the other directory
specifications that were not explicitly provided.
The most portable way to affect installation locations is to pass the
correct locations to `configure'; however, many packages provide one or
both of the following shortcuts of passing variable assignments to the
`make install' command line to change installation locations without
having to reconfigure or recompile.
The first method involves providing an override variable for each
affected directory. For example, `make install
prefix=/alternate/directory' will choose an alternate location for all
directory configuration variables that were expressed in terms of
`${prefix}'. Any directories that were specified during `configure',
but not in terms of `${prefix}', must each be overridden at install
time for the entire installation to be relocated. The approach of
makefile variable overrides for each directory variable is required by
the GNU Coding Standards, and ideally causes no recompilation.
However, some platforms have known limitations with the semantics of
shared libraries that end up requiring recompilation when using this
method, particularly noticeable in packages that use GNU Libtool.
The second method involves providing the `DESTDIR' variable. For
example, `make install DESTDIR=/alternate/directory' will prepend
`/alternate/directory' before all installation names. The approach of
`DESTDIR' overrides is not required by the GNU Coding Standards, and
does not work on platforms that have drive letters. On the other hand,
it does better at avoiding recompilation issues, and works well even
when some directory options were not specified in terms of `${prefix}'
at `configure' time.
Optional Features
=================
If the package supports it, you can cause programs to be installed
with an extra prefix or suffix on their names by giving `configure' the
option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
Some packages pay attention to `--enable-FEATURE' options to
`configure', where FEATURE indicates an optional part of the package.
They may also pay attention to `--with-PACKAGE' options, where PACKAGE
is something like `gnu-as' or `x' (for the X Window System). The
`README' should mention any `--enable-' and `--with-' options that the
package recognizes.
For packages that use the X Window System, `configure' can usually
find the X include and library files automatically, but if it doesn't,
you can use the `configure' options `--x-includes=DIR' and
`--x-libraries=DIR' to specify their locations.
Some packages offer the ability to configure how verbose the
execution of `make' will be. For these packages, running `./configure
--enable-silent-rules' sets the default to minimal output, which can be
overridden with `make V=1'; while running `./configure
--disable-silent-rules' sets the default to verbose, which can be
overridden with `make V=0'.
Particular systems
==================
On HP-UX, the default C compiler is not ANSI C compatible. If GNU
CC is not installed, it is recommended to use the following options in
order to use an ANSI C compiler:
./configure CC="cc -Ae -D_XOPEN_SOURCE=500"
and if that doesn't work, install pre-built binaries of GCC for HP-UX.
HP-UX `make' updates targets which have the same time stamps as
their prerequisites, which makes it generally unusable when shipped
generated files such as `configure' are involved. Use GNU `make'
instead.
On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot
parse its `<wchar.h>' header file. The option `-nodtk' can be used as
a workaround. If GNU CC is not installed, it is therefore recommended
to try
./configure CC="cc"
and if that doesn't work, try
./configure CC="cc -nodtk"
On Solaris, don't put `/usr/ucb' early in your `PATH'. This
directory contains several dysfunctional programs; working variants of
these programs are available in `/usr/bin'. So, if you need `/usr/ucb'
in your `PATH', put it _after_ `/usr/bin'.
On Haiku, software installed for all users goes in `/boot/common',
not `/usr/local'. It is recommended to use the following options:
./configure --prefix=/boot/common
Specifying the System Type
==========================
There may be some features `configure' cannot figure out
automatically, but needs to determine by the type of machine the package
will run on. Usually, assuming the package is built to be run on the
_same_ architectures, `configure' can figure that out, but if it prints
a message saying it cannot guess the machine type, give it the
`--build=TYPE' option. TYPE can either be a short name for the system
type, such as `sun4', or a canonical name which has the form:
CPU-COMPANY-SYSTEM
where SYSTEM can have one of these forms:
OS
KERNEL-OS
See the file `config.sub' for the possible values of each field. If
`config.sub' isn't included in this package, then this package doesn't
need to know the machine type.
If you are _building_ compiler tools for cross-compiling, you should
use the option `--target=TYPE' to select the type of system they will
produce code for.
If you want to _use_ a cross compiler, that generates code for a
platform different from the build platform, you should specify the
"host" platform (i.e., that on which the generated programs will
eventually be run) with `--host=TYPE'.
Sharing Defaults
================
If you want to set default values for `configure' scripts to share,
you can create a site shell script called `config.site' that gives
default values for variables like `CC', `cache_file', and `prefix'.
`configure' looks for `PREFIX/share/config.site' if it exists, then
`PREFIX/etc/config.site' if it exists. Or, you can set the
`CONFIG_SITE' environment variable to the location of the site script.
A warning: not all `configure' scripts look for a site script.
Defining Variables
==================
Variables not defined in a site shell script can be set in the
environment passed to `configure'. However, some packages may run
configure again during the build, and the customized values of these
variables may be lost. In order to avoid this problem, you should set
them in the `configure' command line, using `VAR=value'. For example:
./configure CC=/usr/local2/bin/gcc
causes the specified `gcc' to be used as the C compiler (unless it is
overridden in the site shell script).
Unfortunately, this technique does not work for `CONFIG_SHELL' due to
an Autoconf bug. Until the bug is fixed you can use this workaround:
CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash
`configure' Invocation
======================
`configure' recognizes the following options to control how it
operates.
`--help'
`-h'
Print a summary of all of the options to `configure', and exit.
`--help=short'
`--help=recursive'
Print a summary of the options unique to this package's
`configure', and exit. The `short' variant lists options used
only in the top level, while the `recursive' variant lists options
also present in any nested packages.
`--version'
`-V'
Print the version of Autoconf used to generate the `configure'
script, and exit.
`--cache-file=FILE'
Enable the cache: use and save the results of the tests in FILE,
traditionally `config.cache'. FILE defaults to `/dev/null' to
disable caching.
`--config-cache'
`-C'
Alias for `--cache-file=config.cache'.
`--quiet'
`--silent'
`-q'
Do not print messages saying which checks are being made. To
suppress all normal output, redirect it to `/dev/null' (any error
messages will still be shown).
`--srcdir=DIR'
Look for the package's source code in directory DIR. Usually
`configure' can determine that directory automatically.
`--prefix=DIR'
Use DIR as the installation prefix. *note Installation Names::
for more details, including other options available for fine-tuning
the installation locations.
`--no-create'
`-n'
Run the configure checks, but stop before creating any output
files.
`configure' also accepts some other, not widely useful, options. Run
`configure --help' for more details.
-21
View File
@@ -1,21 +0,0 @@
AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
lib_LTLIBRARIES = libsqlite3.la
libsqlite3_la_SOURCES = sqlite3.c
libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
bin_PROGRAMS = sqlite3
sqlite3_SOURCES = shell.c sqlite3.h
EXTRA_sqlite3_SOURCES = sqlite3.c
sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB
include_HEADERS = sqlite3.h sqlite3ext.h
EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
pkgconfigdir = ${libdir}/pkgconfig
pkgconfig_DATA = sqlite3.pc
man_MANS = sqlite3.1
File diff suppressed because it is too large Load Diff
-11
View File
@@ -1,11 +0,0 @@
This directory contains components use to build an autoconf-ready package
of the SQLite amalgamation: sqlite-autoconf-30XXXXXX.tar.gz
To build the autoconf amalgamation, run from the top-level:
./configure
make amalgamation-tarball
The amalgamation-tarball target (also available in "main.mk") runs the
script tool/mkautoconfamal.sh which does the work. Refer to that script
for details.
-113
View File
@@ -1,113 +0,0 @@
This package contains:
* the SQLite library amalgamation source code file: sqlite3.c
* the sqlite3.h and sqlite3ext.h header files that define the C-language
interface to the sqlite3.c library file
* the shell.c file used to build the sqlite3 command-line shell program
* autoconf/automake installation infrastucture for building on POSIX
compliant systems
* a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft
Visual C++ on Windows
SUMMARY OF HOW TO BUILD
=======================
Unix: ./configure; make
Windows: nmake /f Makefile.msc
BUILDING ON POSIX
=================
The generic installation instructions for autoconf/automake are found
in the INSTALL file.
The following SQLite specific boolean options are supported:
--enable-readline use readline in shell tool [default=yes]
--enable-threadsafe build a thread-safe library [default=yes]
--enable-dynamic-extensions support loadable extensions [default=yes]
The default value for the CFLAGS variable (options passed to the C
compiler) includes debugging symbols in the build, resulting in larger
binaries than are necessary. Override it on the configure command
line like this:
$ CFLAGS="-Os" ./configure
to produce a smaller installation footprint.
Other SQLite compilation parameters can also be set using CFLAGS. For
example:
$ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure
BUILDING WITH MICROSOFT VISUAL C++
==================================
To compile for Windows using Microsoft Visual C++:
$ nmake /f Makefile.msc
Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows
platform variants may be built by adding additional macros to the NMAKE
command line.
Building for WinRT 8.0
----------------------
FOR_WINRT=1
Using Microsoft Visual C++ 2012 (or later) is required. When using the
above, something like the following macro will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
Building for WinRT 8.1
----------------------
FOR_WINRT=1
Using Microsoft Visual C++ 2013 (or later) is required. When using the
above, something like the following macro will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
Building for UWP 10.0
---------------------
FOR_WINRT=1 FOR_UWP=1
Using Microsoft Visual C++ 2015 (or later) is required. When using the
above, something like the following macros will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
"PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
"NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
Building for the Windows 10 SDK
-------------------------------
FOR_WIN10=1
Using Microsoft Visual C++ 2015 (or later) is required. When using the
above, no other macros should be needed on the NMAKE command line.
Other preprocessor defines
--------------------------
Additionally, preprocessor defines may be specified by using the OPTS macro
on the NMAKE command line. However, not all possible preprocessor defines
may be specified in this manner as some require the amalgamation to be built
with them enabled (see http://www.sqlite.org/compile.html). For example, the
following will work:
"OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
However, the following will not compile unless the amalgamation was built
with it enabled:
"OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"
-193
View File
@@ -1,193 +0,0 @@
#-----------------------------------------------------------------------
# Supports the following non-standard switches.
#
# --enable-threadsafe
# --enable-readline
# --enable-editline
# --enable-static-shell
# --enable-dynamic-extensions
#
AC_PREREQ(2.61)
AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
AC_CONFIG_SRCDIR([sqlite3.c])
AC_CONFIG_AUX_DIR([.])
# Use automake.
AM_INIT_AUTOMAKE([foreign])
AC_SYS_LARGEFILE
# Check for required programs.
AC_PROG_CC
AC_PROG_LIBTOOL
AC_PROG_MKDIR_P
# Check for library functions that SQLite can optionally use.
AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
AC_FUNC_STRERROR_R
AC_CONFIG_FILES([Makefile sqlite3.pc])
AC_SUBST(BUILD_CFLAGS)
#-------------------------------------------------------------------------
# Two options to enable readline compatible libraries:
#
# --enable-editline
# --enable-readline
#
# Both are enabled by default. If, after command line processing both are
# still enabled, the script searches for editline first and automatically
# disables readline if it is found. So, to use readline explicitly, the
# user must pass "--disable-editline". To disable command line editing
# support altogether, "--disable-editline --disable-readline".
#
# When searching for either library, check for headers before libraries
# as some distros supply packages that contain libraries but not header
# files, which come as a separate development package.
#
AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])])
AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])])
AS_IF([ test x"$enable_editline" != xno ],[
AC_CHECK_HEADERS([editline/readline.h],[
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS([readline],[edit],[
AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
READLINE_LIBS="$LIBS -ltinfo"
enable_readline=no
],[],[-ltinfo])
AS_UNSET(ac_cv_search_readline)
LIBS=$sLIBS
])
])
AS_IF([ test x"$enable_readline" != xno ],[
AC_CHECK_HEADERS([readline/readline.h],[
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], [])
AC_SEARCH_LIBS(readline,[readline edit], [
AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper)
READLINE_LIBS=$LIBS
])
LIBS=$sLIBS
])
])
AC_SUBST(READLINE_LIBS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-threadsafe
#
AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
[--enable-threadsafe], [build a thread-safe library [default=yes]])],
[], [enable_threadsafe=yes])
THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
if test x"$enable_threadsafe" != "xno"; then
THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
AC_SEARCH_LIBS(pthread_create, pthread)
AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
fi
AC_SUBST(THREADSAFE_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-dynamic-extensions
#
AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
[--enable-dynamic-extensions], [support loadable extensions [default=yes]])],
[], [enable_dynamic_extensions=yes])
if test x"$enable_dynamic_extensions" != "xno"; then
AC_SEARCH_LIBS(dlopen, dl)
else
DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
fi
AC_MSG_CHECKING([for whether to support dynamic extensions])
AC_MSG_RESULT($enable_dynamic_extensions)
AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-fts5
#
AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
[--enable-fts5], [include fts5 support [default=no]])],
[], [enable_fts5=no])
if test x"$enable_fts5" = "xyes"; then
AC_SEARCH_LIBS(log, m)
FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
fi
AC_SUBST(FTS5_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-json1
#
AC_ARG_ENABLE(json1, [AS_HELP_STRING(
[--enable-json1], [include json1 support [default=no]])],
[], [enable_json1=no])
if test x"$enable_json1" = "xyes"; then
JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
fi
AC_SUBST(JSON1_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-session
#
AC_ARG_ENABLE(session, [AS_HELP_STRING(
[--enable-session], [enable the session extension [default=no]])],
[], [enable_session=no])
if test x"$enable_session" = "xyes"; then
SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
fi
AC_SUBST(SESSION_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-static-shell
#
AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
[--enable-static-shell],
[statically link libsqlite3 into shell tool [default=yes]])],
[], [enable_static_shell=yes])
if test x"$enable_static_shell" = "xyes"; then
EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
else
EXTRA_SHELL_OBJ=libsqlite3.la
fi
AC_SUBST(EXTRA_SHELL_OBJ)
#-----------------------------------------------------------------------
AC_CHECK_FUNCS(posix_fallocate)
#-----------------------------------------------------------------------
# UPDATE: Maybe it's better if users just set CFLAGS before invoking
# configure. This option doesn't really add much...
#
# --enable-tempstore
#
# AC_ARG_ENABLE(tempstore, [AS_HELP_STRING(
# [--enable-tempstore],
# [in-memory temporary tables (never, no, yes, always) [default=no]])],
# [], [enable_tempstore=no])
# AC_MSG_CHECKING([for whether or not to store temp tables in-memory])
# case "$enable_tempstore" in
# never ) TEMP_STORE=0 ;;
# no ) TEMP_STORE=1 ;;
# always ) TEMP_STORE=3 ;;
# yes ) TEMP_STORE=3 ;;
# * )
# TEMP_STORE=1
# enable_tempstore=yes
# ;;
# esac
# AC_MSG_RESULT($enable_tempstore)
# AC_SUBST(TEMP_STORE)
#-----------------------------------------------------------------------
AC_OUTPUT
-440
View File
@@ -1,440 +0,0 @@
# Makefile.in --
#
# This file is a Makefile for Sample TEA Extension. If it has the name
# "Makefile.in" then it is a template for a Makefile; to generate the
# actual Makefile, run "./configure", which is a configuration script
# generated by the "autoconf" program (constructs like "@foo@" will get
# replaced in the actual Makefile.
#
# Copyright (c) 1999 Scriptics Corporation.
# Copyright (c) 2002-2005 ActiveState Corporation.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: Makefile.in,v 1.59 2005/07/26 19:17:02 mdejong Exp $
#========================================================================
# Add additional lines to handle any additional AC_SUBST cases that
# have been added in a customized configure script.
#========================================================================
#SAMPLE_NEW_VAR = @SAMPLE_NEW_VAR@
#========================================================================
# Nothing of the variables below this line should need to be changed.
# Please check the TARGETS section below to make sure the make targets
# are correct.
#========================================================================
#========================================================================
# The names of the source files is defined in the configure script.
# The object files are used for linking into the final library.
# This will be used when a dist target is added to the Makefile.
# It is not important to specify the directory, as long as it is the
# $(srcdir) or in the generic, win or unix subdirectory.
#========================================================================
PKG_SOURCES = @PKG_SOURCES@
PKG_OBJECTS = @PKG_OBJECTS@
PKG_STUB_SOURCES = @PKG_STUB_SOURCES@
PKG_STUB_OBJECTS = @PKG_STUB_OBJECTS@
#========================================================================
# PKG_TCL_SOURCES identifies Tcl runtime files that are associated with
# this package that need to be installed, if any.
#========================================================================
PKG_TCL_SOURCES = @PKG_TCL_SOURCES@
#========================================================================
# This is a list of public header files to be installed, if any.
#========================================================================
PKG_HEADERS = @PKG_HEADERS@
#========================================================================
# "PKG_LIB_FILE" refers to the library (dynamic or static as per
# configuration options) composed of the named objects.
#========================================================================
PKG_LIB_FILE = @PKG_LIB_FILE@
PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@
lib_BINARIES = $(PKG_LIB_FILE)
BINARIES = $(lib_BINARIES)
SHELL = @SHELL@
srcdir = @srcdir@
prefix = @prefix@
exec_prefix = @exec_prefix@
bindir = @bindir@
libdir = @libdir@
datarootdir = @datarootdir@
datadir = @datadir@
mandir = @mandir@
includedir = @includedir@
DESTDIR =
PKG_DIR = $(PACKAGE_NAME)$(PACKAGE_VERSION)
pkgdatadir = $(datadir)/$(PKG_DIR)
pkglibdir = $(libdir)/$(PKG_DIR)
pkgincludedir = $(includedir)/$(PKG_DIR)
top_builddir = .
INSTALL = @INSTALL@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
CC = @CC@
CFLAGS_DEFAULT = @CFLAGS_DEFAULT@
CFLAGS_WARNING = @CFLAGS_WARNING@
CLEANFILES = @CLEANFILES@
EXEEXT = @EXEEXT@
LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@
MAKE_LIB = @MAKE_LIB@
MAKE_SHARED_LIB = @MAKE_SHARED_LIB@
MAKE_STATIC_LIB = @MAKE_STATIC_LIB@
MAKE_STUB_LIB = @MAKE_STUB_LIB@
OBJEXT = @OBJEXT@
RANLIB = @RANLIB@
RANLIB_STUB = @RANLIB_STUB@
SHLIB_CFLAGS = @SHLIB_CFLAGS@
SHLIB_LD = @SHLIB_LD@
SHLIB_LD_LIBS = @SHLIB_LD_LIBS@
STLIB_LD = @STLIB_LD@
#TCL_DEFS = @TCL_DEFS@
TCL_BIN_DIR = @TCL_BIN_DIR@
TCL_SRC_DIR = @TCL_SRC_DIR@
#TK_BIN_DIR = @TK_BIN_DIR@
#TK_SRC_DIR = @TK_SRC_DIR@
# This is no longer necessary even for packages that use private Tcl headers
#TCL_TOP_DIR_NATIVE = @TCL_TOP_DIR_NATIVE@
# Not used, but retained for reference of what libs Tcl required
#TCL_LIBS = @TCL_LIBS@
#========================================================================
# TCLLIBPATH seeds the auto_path in Tcl's init.tcl so we can test our
# package without installing. The other environment variables allow us
# to test against an uninstalled Tcl. Add special env vars that you
# require for testing here (like TCLX_LIBRARY).
#========================================================================
EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR)
#EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR)
TCLLIBPATH = $(top_builddir)
TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` \
@LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \
PATH="$(EXTRA_PATH):$(PATH)" \
TCLLIBPATH="$(TCLLIBPATH)"
# TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library`
TCLSH_PROG = @TCLSH_PROG@
TCLSH = $(TCLSH_ENV) $(TCLSH_PROG)
#WISH_PROG = @WISH_PROG@
#WISH = $(TCLSH_ENV) $(WISH_PROG)
SHARED_BUILD = @SHARED_BUILD@
INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I$(srcdir)/..
#INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@
PKG_CFLAGS = @PKG_CFLAGS@
# TCL_DEFS is not strictly need here, but if you remove it, then you
# must make sure that configure.in checks for the necessary components
# that your library may use. TCL_DEFS can actually be a problem if
# you do not compile with a similar machine setup as the Tcl core was
# compiled with.
#DEFS = $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS)
DEFS = @DEFS@ $(PKG_CFLAGS)
CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl
CPPFLAGS = @CPPFLAGS@
LIBS = @PKG_LIBS@ @LIBS@
AR = @AR@
CFLAGS = @CFLAGS@
COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
#========================================================================
# Start of user-definable TARGETS section
#========================================================================
#========================================================================
# TEA TARGETS. Please note that the "libraries:" target refers to platform
# independent files, and the "binaries:" target inclues executable programs and
# platform-dependent libraries. Modify these targets so that they install
# the various pieces of your package. The make and install rules
# for the BINARIES that you specified above have already been done.
#========================================================================
all: binaries libraries doc
#========================================================================
# The binaries target builds executable programs, Windows .dll's, unix
# shared/static libraries, and any other platform-dependent files.
# The list of targets to build for "binaries:" is specified at the top
# of the Makefile, in the "BINARIES" variable.
#========================================================================
binaries: $(BINARIES)
libraries:
#========================================================================
# Your doc target should differentiate from doc builds (by the developer)
# and doc installs (see install-doc), which just install the docs on the
# end user machine when building from source.
#========================================================================
doc:
@echo "If you have documentation to create, place the commands to"
@echo "build the docs in the 'doc:' target. For example:"
@echo " xml2nroff sample.xml > sample.n"
@echo " xml2html sample.xml > sample.html"
install: all install-binaries install-libraries install-doc
install-binaries: binaries install-lib-binaries install-bin-binaries
#========================================================================
# This rule installs platform-independent files, such as header files.
# The list=...; for p in $$list handles the empty list case x-platform.
#========================================================================
install-libraries: libraries
@mkdir -p $(DESTDIR)$(includedir)
@echo "Installing header files in $(DESTDIR)$(includedir)"
@list='$(PKG_HEADERS)'; for i in $$list; do \
echo "Installing $(srcdir)/$$i" ; \
$(INSTALL_DATA) $(srcdir)/$$i $(DESTDIR)$(includedir) ; \
done;
#========================================================================
# Install documentation. Unix manpages should go in the $(mandir)
# directory.
#========================================================================
install-doc: doc
@mkdir -p $(DESTDIR)$(mandir)/mann
@echo "Installing documentation in $(DESTDIR)$(mandir)"
@list='$(srcdir)/doc/*.n'; for i in $$list; do \
echo "Installing $$i"; \
rm -f $(DESTDIR)$(mandir)/mann/`basename $$i`; \
$(INSTALL_DATA) $$i $(DESTDIR)$(mandir)/mann ; \
done
test: binaries libraries
@echo "SQLite TEA distribution does not include tests"
shell: binaries libraries
@$(TCLSH) $(SCRIPT)
gdb:
$(TCLSH_ENV) gdb $(TCLSH_PROG) $(SCRIPT)
depend:
#========================================================================
# $(PKG_LIB_FILE) should be listed as part of the BINARIES variable
# mentioned above. That will ensure that this target is built when you
# run "make binaries".
#
# The $(PKG_OBJECTS) objects are created and linked into the final
# library. In most cases these object files will correspond to the
# source files above.
#========================================================================
$(PKG_LIB_FILE): $(PKG_OBJECTS)
-rm -f $(PKG_LIB_FILE)
${MAKE_LIB}
$(RANLIB) $(PKG_LIB_FILE)
$(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS)
-rm -f $(PKG_STUB_LIB_FILE)
${MAKE_STUB_LIB}
$(RANLIB_STUB) $(PKG_STUB_LIB_FILE)
#========================================================================
# We need to enumerate the list of .c to .o lines here.
#
# In the following lines, $(srcdir) refers to the toplevel directory
# containing your extension. If your sources are in a subdirectory,
# you will have to modify the paths to reflect this:
#
# sample.$(OBJEXT): $(srcdir)/generic/sample.c
# $(COMPILE) -c `@CYGPATH@ $(srcdir)/generic/sample.c` -o $@
#
# Setting the VPATH variable to a list of paths will cause the makefile
# to look into these paths when resolving .c to .obj dependencies.
# As necessary, add $(srcdir):$(srcdir)/compat:....
#========================================================================
VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win
.c.@OBJEXT@:
$(COMPILE) -c `@CYGPATH@ $<` -o $@
#========================================================================
# Distribution creation
# You may need to tweak this target to make it work correctly.
#========================================================================
#COMPRESS = tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar
COMPRESS = gtar zcvf $(PKG_DIR).tar.gz $(PKG_DIR)
DIST_ROOT = /tmp/dist
DIST_DIR = $(DIST_ROOT)/$(PKG_DIR)
dist-clean:
rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.*
dist: dist-clean
mkdir -p $(DIST_DIR)
cp -p $(srcdir)/README* $(srcdir)/license* \
$(srcdir)/aclocal.m4 $(srcdir)/configure $(srcdir)/*.in \
$(DIST_DIR)/
chmod 664 $(DIST_DIR)/Makefile.in $(DIST_DIR)/aclocal.m4
chmod 775 $(DIST_DIR)/configure $(DIST_DIR)/configure.in
for i in $(srcdir)/*.[ch]; do \
if [ -f $$i ]; then \
cp -p $$i $(DIST_DIR)/ ; \
fi; \
done;
mkdir $(DIST_DIR)/tclconfig
cp $(srcdir)/tclconfig/install-sh $(srcdir)/tclconfig/tcl.m4 \
$(DIST_DIR)/tclconfig/
chmod 664 $(DIST_DIR)/tclconfig/tcl.m4
chmod +x $(DIST_DIR)/tclconfig/install-sh
list='demos doc generic library mac tests unix win'; \
for p in $$list; do \
if test -d $(srcdir)/$$p ; then \
mkdir $(DIST_DIR)/$$p; \
cp -p $(srcdir)/$$p/*.* $(DIST_DIR)/$$p/; \
fi; \
done
(cd $(DIST_ROOT); $(COMPRESS);)
#========================================================================
# End of user-definable section
#========================================================================
#========================================================================
# Don't modify the file to clean here. Instead, set the "CLEANFILES"
# variable in configure.in
#========================================================================
clean:
-test -z "$(BINARIES)" || rm -f $(BINARIES)
-rm -f *.$(OBJEXT) core *.core
-test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
distclean: clean
-rm -f *.tab.c
-rm -f $(CONFIG_CLEAN_FILES)
-rm -f config.h config.cache config.log config.status
#========================================================================
# Install binary object libraries. On Windows this includes both .dll and
# .lib files. Because the .lib files are not explicitly listed anywhere,
# we need to deduce their existence from the .dll file of the same name.
# Library files go into the lib directory.
# In addition, this will generate the pkgIndex.tcl
# file in the install location (assuming it can find a usable tclsh shell)
#
# You should not have to modify this target.
#========================================================================
install-lib-binaries: binaries
@mkdir -p $(DESTDIR)$(pkglibdir)
@list='$(lib_BINARIES)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p; \
stub=`echo $$p|sed -e "s/.*\(stub\).*/\1/"`; \
if test "x$$stub" = "xstub"; then \
echo " $(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p"; \
$(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p; \
else \
echo " $(RANLIB) $(DESTDIR)$(pkglibdir)/$$p"; \
$(RANLIB) $(DESTDIR)$(pkglibdir)/$$p; \
fi; \
ext=`echo $$p|sed -e "s/.*\.//"`; \
if test "x$$ext" = "xdll"; then \
lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \
if test -f $$lib; then \
echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \
$(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib; \
fi; \
fi; \
fi; \
done
@list='$(PKG_TCL_SOURCES)'; for p in $$list; do \
if test -f $(srcdir)/$$p; then \
destp=`basename $$p`; \
echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \
$(INSTALL_DATA) $(srcdir)/$$p $(DESTDIR)$(pkglibdir)/$$destp; \
fi; \
done
@if test "x$(SHARED_BUILD)" = "x1"; then \
echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \
$(INSTALL_DATA) pkgIndex.tcl $(DESTDIR)$(pkglibdir); \
fi
#========================================================================
# Install binary executables (e.g. .exe files and dependent .dll files)
# This is for files that must go in the bin directory (located next to
# wish and tclsh), like dependent .dll files on Windows.
#
# You should not have to modify this target, except to define bin_BINARIES
# above if necessary.
#========================================================================
install-bin-binaries: binaries
@mkdir -p $(DESTDIR)$(bindir)
@list='$(bin_BINARIES)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p; \
fi; \
done
.SUFFIXES: .c .$(OBJEXT)
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
uninstall-binaries:
list='$(lib_BINARIES)'; for p in $$list; do \
rm -f $(DESTDIR)$(pkglibdir)/$$p; \
done
list='$(PKG_TCL_SOURCES)'; for p in $$list; do \
p=`basename $$p`; \
rm -f $(DESTDIR)$(pkglibdir)/$$p; \
done
list='$(bin_BINARIES)'; for p in $$list; do \
rm -f $(DESTDIR)$(bindir)/$$p; \
done
.PHONY: all binaries clean depend distclean doc install libraries test
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:
-36
View File
@@ -1,36 +0,0 @@
This is the SQLite extension for Tcl using the Tcl Extension
Architecture (TEA). For additional information on SQLite see
http://www.sqlite.org/
UNIX BUILD
==========
Building under most UNIX systems is easy, just run the configure script
and then run make. For more information about the build process, see
the tcl/unix/README file in the Tcl src dist. The following minimal
example will install the extension in the /opt/tcl directory.
$ cd sqlite-*-tea
$ ./configure --prefix=/opt/tcl
$ make
$ make install
WINDOWS BUILD
=============
The recommended method to build extensions under windows is to use the
Msys + Mingw build process. This provides a Unix-style build while
generating native Windows binaries. Using the Msys + Mingw build tools
means that you can use the same configure script as per the Unix build
to create a Makefile. See the tcl/win/README file for the URL of
the Msys + Mingw download.
If you have VC++ then you may wish to use the files in the win
subdirectory and build the extension using just VC++. These files have
been designed to be as generic as possible but will require some
additional maintenance by the project developer to synchronise with
the TEA configure.in and Makefile.in files. Instructions for using the
VC++ makefile are written in the first part of the Makefile.vc
file.
-9
View File
@@ -1,9 +0,0 @@
#
# Include the TEA standard macro set
#
builtin(include,tclconfig/tcl.m4)
#
# Add here whatever m4 macros you want to define for your package
#
-201
View File
@@ -1,201 +0,0 @@
#!/bin/bash -norc
dnl This file is an input file used by the GNU "autoconf" program to
dnl generate the file "configure", which is run during Tcl installation
dnl to configure the system for the local environment.
#
# RCS: @(#) $Id: configure.in,v 1.43 2005/07/26 19:17:05 mdejong Exp $
#-----------------------------------------------------------------------
# Sample configure.in for Tcl Extensions. The only places you should
# need to modify this file are marked by the string __CHANGE__
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# __CHANGE__
# Set your package name and version numbers here.
#
# This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION
# set as provided. These will also be added as -D defs in your Makefile
# so you can encode the package version directly into the source files.
#-----------------------------------------------------------------------
AC_INIT([sqlite], [3.7.4])
#--------------------------------------------------------------------
# Call TEA_INIT as the first TEA_ macro to set up initial vars.
# This will define a ${TEA_PLATFORM} variable == "unix" or "windows"
# as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE.
#--------------------------------------------------------------------
TEA_INIT([3.9])
AC_CONFIG_AUX_DIR(tclconfig)
#--------------------------------------------------------------------
# Load the tclConfig.sh file
#--------------------------------------------------------------------
TEA_PATH_TCLCONFIG
TEA_LOAD_TCLCONFIG
#--------------------------------------------------------------------
# Load the tkConfig.sh file if necessary (Tk extension)
#--------------------------------------------------------------------
#TEA_PATH_TKCONFIG
#TEA_LOAD_TKCONFIG
#-----------------------------------------------------------------------
# Handle the --prefix=... option by defaulting to what Tcl gave.
# Must be called after TEA_LOAD_TCLCONFIG and before TEA_SETUP_COMPILER.
#-----------------------------------------------------------------------
TEA_PREFIX
#-----------------------------------------------------------------------
# Standard compiler checks.
# This sets up CC by using the CC env var, or looks for gcc otherwise.
# This also calls AC_PROG_CC, AC_PROG_INSTALL and a few others to create
# the basic setup necessary to compile executables.
#-----------------------------------------------------------------------
TEA_SETUP_COMPILER
#-----------------------------------------------------------------------
# __CHANGE__
# Specify the C source files to compile in TEA_ADD_SOURCES,
# public headers that need to be installed in TEA_ADD_HEADERS,
# stub library C source files to compile in TEA_ADD_STUB_SOURCES,
# and runtime Tcl library files in TEA_ADD_TCL_SOURCES.
# This defines PKG(_STUB)_SOURCES, PKG(_STUB)_OBJECTS, PKG_HEADERS
# and PKG_TCL_SOURCES.
#-----------------------------------------------------------------------
TEA_ADD_SOURCES([tclsqlite3.c])
TEA_ADD_HEADERS([])
TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"])
TEA_ADD_LIBS([])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
TEA_ADD_STUB_SOURCES([])
TEA_ADD_TCL_SOURCES([])
#--------------------------------------------------------------------
# The --with-system-sqlite causes the TCL bindings to SQLite to use
# the system shared library for SQLite rather than statically linking
# against its own private copy. This is dangerous and leads to
# undersirable dependences and is not recommended.
# Patchs from rmax.
#--------------------------------------------------------------------
AC_ARG_WITH([system-sqlite],
[AC_HELP_STRING([--with-system-sqlite],
[use a system-supplied libsqlite3 instead of the bundled one])],
[], [with_system_sqlite=no])
if test x$with_system_sqlite != xno; then
AC_CHECK_HEADER([sqlite3.h],
[AC_CHECK_LIB([sqlite3],[sqlite3_initialize],
[AC_DEFINE(USE_SYSTEM_SQLITE)
LIBS="$LIBS -lsqlite3"])])
fi
#--------------------------------------------------------------------
# __CHANGE__
# Choose which headers you need. Extension authors should try very
# hard to only rely on the Tcl public header files. Internal headers
# contain private data structures and are subject to change without
# notice.
# This MUST be called after TEA_LOAD_TCLCONFIG / TEA_LOAD_TKCONFIG
#--------------------------------------------------------------------
TEA_PUBLIC_TCL_HEADERS
#TEA_PRIVATE_TCL_HEADERS
#TEA_PUBLIC_TK_HEADERS
#TEA_PRIVATE_TK_HEADERS
#TEA_PATH_X
#--------------------------------------------------------------------
# Check whether --enable-threads or --disable-threads was given.
# This auto-enables if Tcl was compiled threaded.
#--------------------------------------------------------------------
TEA_ENABLE_THREADS
if test "${TCL_THREADS}" = "1" ; then
AC_DEFINE(SQLITE_THREADSAFE, 1, [Trigger sqlite threadsafe build])
# Not automatically added by Tcl because its assumed Tcl links to them,
# but it may not if it isn't really a threaded build.
TEA_ADD_LIBS([$THREADS_LIBS])
else
AC_DEFINE(SQLITE_THREADSAFE, 0, [Trigger sqlite non-threadsafe build])
fi
#--------------------------------------------------------------------
# The statement below defines a collection of symbols related to
# building as a shared library instead of a static library.
#--------------------------------------------------------------------
TEA_ENABLE_SHARED
#--------------------------------------------------------------------
# This macro figures out what flags to use with the compiler/linker
# when building shared/static debug/optimized objects. This information
# can be taken from the tclConfig.sh file, but this figures it all out.
#--------------------------------------------------------------------
TEA_CONFIG_CFLAGS
#--------------------------------------------------------------------
# Set the default compiler switches based on the --enable-symbols option.
#--------------------------------------------------------------------
TEA_ENABLE_SYMBOLS
#--------------------------------------------------------------------
# Everyone should be linking against the Tcl stub library. If you
# can't for some reason, remove this definition. If you aren't using
# stubs, you also need to modify the SHLIB_LD_LIBS setting below to
# link against the non-stubbed Tcl library. Add Tk too if necessary.
#--------------------------------------------------------------------
AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs])
#AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
#--------------------------------------------------------------------
# Redefine fdatasync as fsync on systems that lack fdatasync
#--------------------------------------------------------------------
#
#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync))
# Check for library functions that SQLite can optionally use.
AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
AC_FUNC_STRERROR_R
#--------------------------------------------------------------------
# This macro generates a line to use when building a library. It
# depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS,
# and TEA_LOAD_TCLCONFIG macros above.
#--------------------------------------------------------------------
TEA_MAKE_LIB
#--------------------------------------------------------------------
# Determine the name of the tclsh and/or wish executables in the
# Tcl and Tk build directories or the location they were installed
# into. These paths are used to support running test cases only,
# the Makefile should not be making use of these paths to generate
# a pkgIndex.tcl file or anything else at extension build time.
#--------------------------------------------------------------------
TEA_PROG_TCLSH
#TEA_PROG_WISH
#--------------------------------------------------------------------
# Finally, substitute all of the various values into the Makefile.
# You may alternatively have a special pkgIndex.tcl.in or other files
# which require substituting th AC variables in. Include these here.
#--------------------------------------------------------------------
AC_OUTPUT([Makefile pkgIndex.tcl])
-15
View File
@@ -1,15 +0,0 @@
.TH sqlite3 n 4.1 "Tcl-Extensions"
.HS sqlite3 tcl
.BS
.SH NAME
sqlite3 \- an interface to the SQLite3 database engine
.SH SYNOPSIS
\fBsqlite3\fI command_name ?filename?\fR
.br
.SH DESCRIPTION
SQLite3 is a self-contains, zero-configuration, transactional SQL database
engine. This extension provides an easy to use interface for accessing
SQLite database files from Tcl.
.PP
For full documentation see \fIhttp://www.sqlite.org/\fR and
in particular \fIhttp://www.sqlite.org/tclsqlite.html\fR.
-6
View File
@@ -1,6 +0,0 @@
The author disclaims copyright to this source code. In place of
a legal notice, here is a blessing:
May you do good and not evil.
May you find forgiveness for yourself and forgive others.
May you share freely, never taking more than you give.
-7
View File
@@ -1,7 +0,0 @@
#
# Tcl package index file
#
# Note sqlite*3* init specifically
#
package ifneeded sqlite3 @PACKAGE_VERSION@ \
[list load [file join $dir @PKG_LIB_FILE@] Sqlite3]
-528
View File
@@ -1,528 +0,0 @@
#!/bin/sh
# install - install a program, script, or datafile
scriptversion=2011-04-20.01; # UTC
# This originates from X11R5 (mit/util/scripts/install.sh), which was
# later released in X11R6 (xc/config/util/install.sh) with the
# following copyright and license.
#
# Copyright (C) 1994 X Consortium
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# Except as contained in this notice, the name of the X Consortium shall not
# be used in advertising or otherwise to promote the sale, use or other deal-
# ings in this Software without prior written authorization from the X Consor-
# tium.
#
#
# FSF changes to this file are in the public domain.
#
# Calling this script install-sh is preferred over install.sh, to prevent
# `make' implicit rules from creating a file called install from it
# when there is no Makefile.
#
# This script is compatible with the BSD install script, but was written
# from scratch.
nl='
'
IFS=" "" $nl"
# set DOITPROG to echo to test this script
# Don't use :- since 4.3BSD and earlier shells don't like it.
doit=${DOITPROG-}
if test -z "$doit"; then
doit_exec=exec
else
doit_exec=$doit
fi
# Put in absolute file names if you don't have them in your path;
# or use environment vars.
chgrpprog=${CHGRPPROG-chgrp}
chmodprog=${CHMODPROG-chmod}
chownprog=${CHOWNPROG-chown}
cmpprog=${CMPPROG-cmp}
cpprog=${CPPROG-cp}
mkdirprog=${MKDIRPROG-mkdir}
mvprog=${MVPROG-mv}
rmprog=${RMPROG-rm}
stripprog=${STRIPPROG-strip}
posix_glob='?'
initialize_posix_glob='
test "$posix_glob" != "?" || {
if (set -f) 2>/dev/null; then
posix_glob=
else
posix_glob=:
fi
}
'
posix_mkdir=
# Desired mode of installed file.
mode=0755
chgrpcmd=
chmodcmd=$chmodprog
chowncmd=
mvcmd=$mvprog
rmcmd="$rmprog -f"
stripcmd=
src=
dst=
dir_arg=
dst_arg=
copy_on_change=false
no_target_directory=
usage="\
Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
or: $0 [OPTION]... SRCFILES... DIRECTORY
or: $0 [OPTION]... -t DIRECTORY SRCFILES...
or: $0 [OPTION]... -d DIRECTORIES...
In the 1st form, copy SRCFILE to DSTFILE.
In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
In the 4th, create DIRECTORIES.
Options:
--help display this help and exit.
--version display version info and exit.
-c (ignored)
-C install only if different (preserve the last data modification time)
-d create directories instead of installing files.
-g GROUP $chgrpprog installed files to GROUP.
-m MODE $chmodprog installed files to MODE.
-o USER $chownprog installed files to USER.
-s $stripprog installed files.
-S $stripprog installed files.
-t DIRECTORY install into DIRECTORY.
-T report an error if DSTFILE is a directory.
Environment variables override the default commands:
CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
RMPROG STRIPPROG
"
while test $# -ne 0; do
case $1 in
-c) ;;
-C) copy_on_change=true;;
-d) dir_arg=true;;
-g) chgrpcmd="$chgrpprog $2"
shift;;
--help) echo "$usage"; exit $?;;
-m) mode=$2
case $mode in
*' '* | *' '* | *'
'* | *'*'* | *'?'* | *'['*)
echo "$0: invalid mode: $mode" >&2
exit 1;;
esac
shift;;
-o) chowncmd="$chownprog $2"
shift;;
-s) stripcmd=$stripprog;;
-S) stripcmd="$stripprog $2"
shift;;
-t) dst_arg=$2
shift;;
-T) no_target_directory=true;;
--version) echo "$0 $scriptversion"; exit $?;;
--) shift
break;;
-*) echo "$0: invalid option: $1" >&2
exit 1;;
*) break;;
esac
shift
done
if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
# When -d is used, all remaining arguments are directories to create.
# When -t is used, the destination is already specified.
# Otherwise, the last argument is the destination. Remove it from $@.
for arg
do
if test -n "$dst_arg"; then
# $@ is not empty: it contains at least $arg.
set fnord "$@" "$dst_arg"
shift # fnord
fi
shift # arg
dst_arg=$arg
done
fi
if test $# -eq 0; then
if test -z "$dir_arg"; then
echo "$0: no input file specified." >&2
exit 1
fi
# It's OK to call `install-sh -d' without argument.
# This can happen when creating conditional directories.
exit 0
fi
if test -z "$dir_arg"; then
do_exit='(exit $ret); exit $ret'
trap "ret=129; $do_exit" 1
trap "ret=130; $do_exit" 2
trap "ret=141; $do_exit" 13
trap "ret=143; $do_exit" 15
# Set umask so as not to create temps with too-generous modes.
# However, 'strip' requires both read and write access to temps.
case $mode in
# Optimize common cases.
*644) cp_umask=133;;
*755) cp_umask=22;;
*[0-7])
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw='% 200'
fi
cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
*)
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw=,u+rw
fi
cp_umask=$mode$u_plus_rw;;
esac
fi
for src
do
# Protect names starting with `-'.
case $src in
-*) src=./$src;;
esac
if test -n "$dir_arg"; then
dst=$src
dstdir=$dst
test -d "$dstdir"
dstdir_status=$?
else
# Waiting for this to be detected by the "$cpprog $src $dsttmp" command
# might cause directories to be created, which would be especially bad
# if $src (and thus $dsttmp) contains '*'.
if test ! -f "$src" && test ! -d "$src"; then
echo "$0: $src does not exist." >&2
exit 1
fi
if test -z "$dst_arg"; then
echo "$0: no destination specified." >&2
exit 1
fi
dst=$dst_arg
# Protect names starting with `-'.
case $dst in
-*) dst=./$dst;;
esac
# If destination is a directory, append the input filename; won't work
# if double slashes aren't ignored.
if test -d "$dst"; then
if test -n "$no_target_directory"; then
echo "$0: $dst_arg: Is a directory" >&2
exit 1
fi
dstdir=$dst
dst=$dstdir/`basename "$src"`
dstdir_status=0
else
# Prefer dirname, but fall back on a substitute if dirname fails.
dstdir=`
(dirname "$dst") 2>/dev/null ||
expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
X"$dst" : 'X\(//\)[^/]' \| \
X"$dst" : 'X\(//\)$' \| \
X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
echo X"$dst" |
sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
s//\1/
q
}
/^X\(\/\/\)[^/].*/{
s//\1/
q
}
/^X\(\/\/\)$/{
s//\1/
q
}
/^X\(\/\).*/{
s//\1/
q
}
s/.*/./; q'
`
test -d "$dstdir"
dstdir_status=$?
fi
fi
obsolete_mkdir_used=false
if test $dstdir_status != 0; then
case $posix_mkdir in
'')
# Create intermediate dirs using mode 755 as modified by the umask.
# This is like FreeBSD 'install' as of 1997-10-28.
umask=`umask`
case $stripcmd.$umask in
# Optimize common cases.
*[2367][2367]) mkdir_umask=$umask;;
.*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
*[0-7])
mkdir_umask=`expr $umask + 22 \
- $umask % 100 % 40 + $umask % 20 \
- $umask % 10 % 4 + $umask % 2
`;;
*) mkdir_umask=$umask,go-w;;
esac
# With -d, create the new directory with the user-specified mode.
# Otherwise, rely on $mkdir_umask.
if test -n "$dir_arg"; then
mkdir_mode=-m$mode
else
mkdir_mode=
fi
posix_mkdir=false
case $umask in
*[123567][0-7][0-7])
# POSIX mkdir -p sets u+wx bits regardless of umask, which
# is incompatible with FreeBSD 'install' when (umask & 300) != 0.
;;
*)
tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
if (umask $mkdir_umask &&
exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
then
if test -z "$dir_arg" || {
# Check for POSIX incompatibilities with -m.
# HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
# other-writeable bit of parent directory when it shouldn't.
# FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
ls_ld_tmpdir=`ls -ld "$tmpdir"`
case $ls_ld_tmpdir in
d????-?r-*) different_mode=700;;
d????-?--*) different_mode=755;;
*) false;;
esac &&
$mkdirprog -m$different_mode -p -- "$tmpdir" && {
ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
}
}
then posix_mkdir=:
fi
rmdir "$tmpdir/d" "$tmpdir"
else
# Remove any dirs left behind by ancient mkdir implementations.
rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
fi
trap '' 0;;
esac;;
esac
if
$posix_mkdir && (
umask $mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
)
then :
else
# The umask is ridiculous, or mkdir does not conform to POSIX,
# or it failed possibly due to a race condition. Create the
# directory the slow way, step by step, checking for races as we go.
case $dstdir in
/*) prefix='/';;
-*) prefix='./';;
*) prefix='';;
esac
eval "$initialize_posix_glob"
oIFS=$IFS
IFS=/
$posix_glob set -f
set fnord $dstdir
shift
$posix_glob set +f
IFS=$oIFS
prefixes=
for d
do
test -z "$d" && continue
prefix=$prefix$d
if test -d "$prefix"; then
prefixes=
else
if $posix_mkdir; then
(umask=$mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
# Don't fail if two instances are running concurrently.
test -d "$prefix" || exit 1
else
case $prefix in
*\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
*) qprefix=$prefix;;
esac
prefixes="$prefixes '$qprefix'"
fi
fi
prefix=$prefix/
done
if test -n "$prefixes"; then
# Don't fail if two instances are running concurrently.
(umask $mkdir_umask &&
eval "\$doit_exec \$mkdirprog $prefixes") ||
test -d "$dstdir" || exit 1
obsolete_mkdir_used=true
fi
fi
fi
if test -n "$dir_arg"; then
{ test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
{ test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
else
# Make a couple of temp file names in the proper directory.
dsttmp=$dstdir/_inst.$$_
rmtmp=$dstdir/_rm.$$_
# Trap to clean up those temp files at exit.
trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
# Copy the file name to the temp name.
(umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
# and set any options; do chmod last to preserve setuid bits.
#
# If any of these fail, we abort the whole thing. If we want to
# ignore errors from any of these, just make sure not to ignore
# errors from the above "$doit $cpprog $src $dsttmp" command.
#
{ test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
{ test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
{ test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
# If -C, don't bother to copy if it wouldn't change the file.
if $copy_on_change &&
old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
eval "$initialize_posix_glob" &&
$posix_glob set -f &&
set X $old && old=:$2:$4:$5:$6 &&
set X $new && new=:$2:$4:$5:$6 &&
$posix_glob set +f &&
test "$old" = "$new" &&
$cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
then
rm -f "$dsttmp"
else
# Rename the file to the real destination.
$doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
# The rename failed, perhaps because mv can't rename something else
# to itself, or perhaps because mv is so ancient that it does not
# support -f.
{
# Now remove or move aside any old file at destination location.
# We try this two ways since rm can't unlink itself on some
# systems and the destination file might be busy for other
# reasons. In this case, the final cleanup might fail but the new
# file should still install successfully.
{
test ! -f "$dst" ||
$doit $rmcmd -f "$dst" 2>/dev/null ||
{ $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
{ $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
} ||
{ echo "$0: cannot unlink or rename $dst" >&2
(exit 1); exit 1
}
} &&
# Now rename the file to the real destination.
$doit $mvcmd "$dsttmp" "$dst"
}
fi || exit 1
trap '' 0
fi
done
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
File diff suppressed because it is too large Load Diff
-414
View File
@@ -1,414 +0,0 @@
# makefile.vc -- -*- Makefile -*-
#
# Microsoft Visual C++ makefile for use with nmake.exe v1.62+ (VC++ 5.0+)
#
# This makefile is based upon the Tcl 8.4 Makefile.vc and modified to
# make it suitable as a general package makefile. Look for the word EDIT
# which marks sections that may need modification. As a minumum you will
# need to change the PROJECT, DOTVERSION and DLLOBJS variables to values
# relevant to your package.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# Copyright (c) 1995-1996 Sun Microsystems, Inc.
# Copyright (c) 1998-2000 Ajuba Solutions.
# Copyright (c) 2001 ActiveState Corporation.
# Copyright (c) 2001-2002 David Gravereaux.
# Copyright (c) 2003 Pat Thoyts
#
#-------------------------------------------------------------------------
# RCS: @(#)$Id: makefile.vc,v 1.4 2004/07/26 08:22:05 patthoyts Exp $
#-------------------------------------------------------------------------
!if !defined(MSDEVDIR) && !defined(MSVCDIR) && !defined(VCINSTALLDIR) && !defined(MSSDK) && !defined(WINDOWSSDKDIR)
MSG = ^
You will need to run vcvars32.bat from Developer Studio, first, to setup^
the environment. Jump to this line to read the new instructions.
!error $(MSG)
!endif
#------------------------------------------------------------------------------
# HOW TO USE this makefile:
#
# 1) It is now necessary to have %MSVCDir% set in the environment. This is
# used as a check to see if vcvars32.bat had been run prior to running
# nmake or during the installation of Microsoft Visual C++, MSVCDir had
# been set globally and the PATH adjusted. Either way is valid.
#
# You'll need to run vcvars32.bat contained in the MsDev's vc(98)/bin
# directory to setup the proper environment, if needed, for your current
# setup. This is a needed bootstrap requirement and allows the swapping of
# different environments to be easier.
#
# 2) To use the Platform SDK (not expressly needed), run setenv.bat after
# vcvars32.bat according to the instructions for it. This can also turn on
# the 64-bit compiler, if your SDK has it.
#
# 3) Targets are:
# all -- Builds everything.
# <project> -- Builds the project (eg: nmake sample)
# test -- Builds and runs the test suite.
# install -- Installs the built binaries and libraries to $(INSTALLDIR)
# in an appropriate subdirectory.
# clean/realclean/distclean -- varying levels of cleaning.
#
# 4) Macros usable on the commandline:
# INSTALLDIR=<path>
# Sets where to install Tcl from the built binaries.
# C:\Progra~1\Tcl is assumed when not specified.
#
# OPTS=static,msvcrt,staticpkg,threads,symbols,profile,loimpact,none
# Sets special options for the core. The default is for none.
# Any combination of the above may be used (comma separated).
# 'none' will over-ride everything to nothing.
#
# static = Builds a static library of the core instead of a
# dll. The shell will be static (and large), as well.
# msvcrt = Effects the static option only to switch it from
# using libcmt(d) as the C runtime [by default] to
# msvcrt(d). This is useful for static embedding
# support.
# staticpkg = Effects the static option only to switch
# tclshXX.exe to have the dde and reg extension linked
# inside it.
# threads = Turns on full multithreading support.
# thrdalloc = Use the thread allocator (shared global free pool).
# symbols = Adds symbols for step debugging.
# profile = Adds profiling hooks. Map file is assumed.
# loimpact = Adds a flag for how NT treats the heap to keep memory
# in use, low. This is said to impact alloc performance.
#
# STATS=memdbg,compdbg,none
# Sets optional memory and bytecode compiler debugging code added
# to the core. The default is for none. Any combination of the
# above may be used (comma separated). 'none' will over-ride
# everything to nothing.
#
# memdbg = Enables the debugging memory allocator.
# compdbg = Enables byte compilation logging.
#
# MACHINE=(IX86|IA64|ALPHA)
# Set the machine type used for the compiler, linker, and
# resource compiler. This hook is needed to tell the tools
# when alternate platforms are requested. IX86 is the default
# when not specified.
#
# TMP_DIR=<path>
# OUT_DIR=<path>
# Hooks to allow the intermediate and output directories to be
# changed. $(OUT_DIR) is assumed to be
# $(BINROOT)\(Release|Debug) based on if symbols are requested.
# $(TMP_DIR) will de $(OUT_DIR)\<buildtype> by default.
#
# TESTPAT=<file>
# Reads the tests requested to be run from this file.
#
# CFG_ENCODING=encoding
# name of encoding for configuration information. Defaults
# to cp1252
#
# 5) Examples:
#
# Basic syntax of calling nmake looks like this:
# nmake [-nologo] -f makefile.vc [target|macrodef [target|macrodef] [...]]
#
# Standard (no frills)
# c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat
# Setting environment for using Microsoft Visual C++ tools.
# c:\tcl_src\win\>nmake -f makefile.vc all
# c:\tcl_src\win\>nmake -f makefile.vc install INSTALLDIR=c:\progra~1\tcl
#
# Building for Win64
# c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat
# Setting environment for using Microsoft Visual C++ tools.
# c:\tcl_src\win\>c:\progra~1\platfo~1\setenv.bat /pre64 /RETAIL
# Targeting Windows pre64 RETAIL
# c:\tcl_src\win\>nmake -f makefile.vc MACHINE=IA64
#
#------------------------------------------------------------------------------
#==============================================================================
###############################################################################
#------------------------------------------------------------------------------
!if !exist("makefile.vc")
MSG = ^
You must run this makefile only from the directory it is in.^
Please `cd` to its location first.
!error $(MSG)
!endif
#-------------------------------------------------------------------------
# Project specific information (EDIT)
#
# You should edit this with the name and version of your project. This
# information is used to generate the name of the package library and
# it's install location.
#
# For example, the sample extension is going to build sample04.dll and
# would install it into $(INSTALLDIR)\lib\sample04
#
# You need to specify the object files that need to be linked into your
# binary here.
#
#-------------------------------------------------------------------------
PROJECT = sqlite3
!include "rules.vc"
# nmakehelp -V <file> <tag> will search the file for tag, skips until a
# number and returns all character until a character not in [0-9.ab]
# is read.
!if [echo REM = This file is generated from Makefile.vc > versions.vc]
!endif
# get project version from row "AC_INIT([sqlite], [3.7.14])"
!if [echo DOTVERSION = \>> versions.vc] \
&& [nmakehlp -V ..\configure.in AC_INIT >> versions.vc]
!endif
!include "versions.vc"
VERSION = $(DOTVERSION:.=)
STUBPREFIX = $(PROJECT)stub
DLLOBJS = \
$(TMP_DIR)\tclsqlite3.obj
#-------------------------------------------------------------------------
# Target names and paths ( shouldn't need changing )
#-------------------------------------------------------------------------
BINROOT = .
ROOT = ..
PRJIMPLIB = $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib
PRJLIBNAME = $(PROJECT)$(VERSION)$(SUFX).$(EXT)
PRJLIB = $(OUT_DIR)\$(PRJLIBNAME)
PRJSTUBLIBNAME = $(STUBPREFIX)$(VERSION).lib
PRJSTUBLIB = $(OUT_DIR)\$(PRJSTUBLIBNAME)
### Make sure we use backslash only.
PRJ_INSTALL_DIR = $(_INSTALLDIR)\$(PROJECT)$(DOTVERSION)
LIB_INSTALL_DIR = $(PRJ_INSTALL_DIR)
BIN_INSTALL_DIR = $(PRJ_INSTALL_DIR)
DOC_INSTALL_DIR = $(PRJ_INSTALL_DIR)
SCRIPT_INSTALL_DIR = $(PRJ_INSTALL_DIR)
INCLUDE_INSTALL_DIR = $(_TCLDIR)\include
### The following paths CANNOT have spaces in them.
GENERICDIR = $(ROOT)\generic
WINDIR = $(ROOT)\win
LIBDIR = $(ROOT)\library
DOCDIR = $(ROOT)\doc
TOOLSDIR = $(ROOT)\tools
COMPATDIR = $(ROOT)\compat
#---------------------------------------------------------------------
# Compile flags
#---------------------------------------------------------------------
!if !$(DEBUG)
!if $(OPTIMIZING)
### This cranks the optimization level to maximize speed
cdebug = -O2 -Op -Gs
!else
cdebug =
!endif
!else if "$(MACHINE)" == "IA64"
### Warnings are too many, can't support warnings into errors.
cdebug = -Z7 -Od -GZ
!else
cdebug = -Z7 -WX -Od -GZ
!endif
### Declarations common to all compiler options
cflags = -nologo -c -W3 -YX -Fp$(TMP_DIR)^\
!if $(MSVCRT)
!if $(DEBUG)
crt = -MDd
!else
crt = -MD
!endif
!else
!if $(DEBUG)
crt = -MTd
!else
crt = -MT
!endif
!endif
INCLUDES = $(TCL_INCLUDES) -I"$(WINDIR)" -I"$(GENERICDIR)" \
-I"$(ROOT)\.."
BASE_CLFAGS = $(cflags) $(cdebug) $(crt) $(INCLUDES) \
-DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \
-DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1
CON_CFLAGS = $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1
TCL_CFLAGS = -DBUILD_sqlite -DUSE_TCL_STUBS \
-DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \
$(OPTDEFINES)
#---------------------------------------------------------------------
# Link flags
#---------------------------------------------------------------------
!if $(DEBUG)
ldebug = -debug:full -debugtype:cv
!else
ldebug = -release -opt:ref -opt:icf,3
!endif
### Declarations common to all linker options
lflags = -nologo -machine:$(MACHINE) $(ldebug)
!if $(PROFILE)
lflags = $(lflags) -profile
!endif
!if $(ALIGN98_HACK) && !$(STATIC_BUILD)
### Align sections for PE size savings.
lflags = $(lflags) -opt:nowin98
!else if !$(ALIGN98_HACK) && $(STATIC_BUILD)
### Align sections for speed in loading by choosing the virtual page size.
lflags = $(lflags) -align:4096
!endif
!if $(LOIMPACT)
lflags = $(lflags) -ws:aggressive
!endif
dlllflags = $(lflags) -dll
conlflags = $(lflags) -subsystem:console
guilflags = $(lflags) -subsystem:windows
baselibs = $(TCLSTUBLIB)
#---------------------------------------------------------------------
# TclTest flags
#---------------------------------------------------------------------
!IF "$(TESTPAT)" != ""
TESTFLAGS = $(TESTFLAGS) -file $(TESTPAT)
!ENDIF
#---------------------------------------------------------------------
# Project specific targets (EDIT)
#---------------------------------------------------------------------
all: setup $(PROJECT)
$(PROJECT): setup $(PRJLIB)
install: install-binaries install-libraries install-docs
# Tests need to ensure we load the right dll file we
# have to handle the output differently on Win9x.
#
!if "$(OS)" == "Windows_NT" || "$(MSVCDIR)" == "IDE"
test: setup $(PROJECT)
set TCL_LIBRARY=$(ROOT)/library
$(TCLSH) <<
load $(PRJLIB:\=/)
cd "$(ROOT)/tests"
set argv "$(TESTFLAGS)"
source all.tcl
<<
!else
test: setup $(PROJECT)
echo Please wait while the test results are collected
set TCL_LIBRARY=$(ROOT)/library
$(TCLSH) << >tests.log
load $(PRJLIB:\=/)
cd "$(ROOT)/tests"
set argv "$(TESTFLAGS)"
source all.tcl
<<
type tests.log | more
!endif
setup:
@if not exist $(OUT_DIR)\nul mkdir $(OUT_DIR)
@if not exist $(TMP_DIR)\nul mkdir $(TMP_DIR)
$(PRJLIB): $(DLLOBJS)
$(link32) $(dlllflags) -out:$@ $(baselibs) @<<
$**
<<
-@del $*.exp
$(PRJSTUBLIB): $(PRJSTUBOBJS)
$(lib32) -nologo -out:$@ $(PRJSTUBOBJS)
#---------------------------------------------------------------------
# Implicit rules
#---------------------------------------------------------------------
{$(WINDIR)}.c{$(TMP_DIR)}.obj::
$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
$<
<<
{$(GENERICDIR)}.c{$(TMP_DIR)}.obj::
$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
$<
<<
{$(COMPATDIR)}.c{$(TMP_DIR)}.obj::
$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ @<<
$<
<<
{$(WINDIR)}.rc{$(TMP_DIR)}.res:
$(rc32) -fo $@ -r -i "$(GENERICDIR)" -D__WIN32__ \
!if $(DEBUG)
-d DEBUG \
!endif
!if $(TCL_THREADS)
-d TCL_THREADS \
!endif
!if $(STATIC_BUILD)
-d STATIC_BUILD \
!endif
$<
.SUFFIXES:
.SUFFIXES:.c .rc
#---------------------------------------------------------------------
# Installation. (EDIT)
#
# You may need to modify this section to reflect the final distribution
# of your files and possibly to generate documentation.
#
#---------------------------------------------------------------------
install-binaries:
@echo Installing binaries to '$(SCRIPT_INSTALL_DIR)'
@if not exist "$(SCRIPT_INSTALL_DIR)" mkdir "$(SCRIPT_INSTALL_DIR)"
@$(CPY) $(PRJLIB) "$(SCRIPT_INSTALL_DIR)" >NUL
install-libraries:
@echo Installing libraries to '$(SCRIPT_INSTALL_DIR)'
@if exist $(LIBDIR) $(CPY) $(LIBDIR)\*.tcl "$(SCRIPT_INSTALL_DIR)"
@echo Installing package index in '$(SCRIPT_INSTALL_DIR)'
@type << >"$(SCRIPT_INSTALL_DIR)\pkgIndex.tcl"
package ifneeded $(PROJECT) $(DOTVERSION) \
[list load [file join $$dir $(PRJLIBNAME)] sqlite3]
<<
install-docs:
@echo Installing documentation files to '$(DOC_INSTALL_DIR)'
@if exist $(DOCDIR) $(CPY) $(DOCDIR)\*.n "$(DOC_INSTALL_DIR)"
#---------------------------------------------------------------------
# Clean up
#---------------------------------------------------------------------
clean:
@if exist $(TMP_DIR)\nul $(RMDIR) $(TMP_DIR)
@if exist $(WINDIR)\version.vc del $(WINDIR)\version.vc
realclean: clean
@if exist $(OUT_DIR)\nul $(RMDIR) $(OUT_DIR)
distclean: realclean
@if exist $(WINDIR)\nmakehlp.exe del $(WINDIR)\nmakehlp.exe
@if exist $(WINDIR)\nmakehlp.obj del $(WINDIR)\nmakehlp.obj
-694
View File
@@ -1,694 +0,0 @@
/*
* ----------------------------------------------------------------------------
* nmakehlp.c --
*
* This is used to fix limitations within nmake and the environment.
*
* Copyright (c) 2002 by David Gravereaux.
* Copyright (c) 2006 by Pat Thoyts
*
* See the file "license.terms" for information on usage and redistribution of
* this file, and for a DISCLAIMER OF ALL WARRANTIES.
* ----------------------------------------------------------------------------
*/
#define _CRT_SECURE_NO_DEPRECATE
#include <windows.h>
#define NO_SHLWAPI_GDI
#define NO_SHLWAPI_STREAM
#define NO_SHLWAPI_REG
#include <shlwapi.h>
#pragma comment (lib, "user32.lib")
#pragma comment (lib, "kernel32.lib")
#pragma comment (lib, "shlwapi.lib")
#include <stdio.h>
#include <math.h>
/*
* This library is required for x64 builds with _some_ versions of MSVC
*/
#if defined(_M_IA64) || defined(_M_AMD64)
#if _MSC_VER >= 1400 && _MSC_VER < 1500
#pragma comment(lib, "bufferoverflowU")
#endif
#endif
/* ISO hack for dumb VC++ */
#ifdef _MSC_VER
#define snprintf _snprintf
#endif
/* protos */
static int CheckForCompilerFeature(const char *option);
static int CheckForLinkerFeature(const char *option);
static int IsIn(const char *string, const char *substring);
static int SubstituteFile(const char *substs, const char *filename);
static int QualifyPath(const char *path);
static const char *GetVersionFromFile(const char *filename, const char *match);
static DWORD WINAPI ReadFromPipe(LPVOID args);
/* globals */
#define CHUNK 25
#define STATICBUFFERSIZE 1000
typedef struct {
HANDLE pipe;
char buffer[STATICBUFFERSIZE];
} pipeinfo;
pipeinfo Out = {INVALID_HANDLE_VALUE, '\0'};
pipeinfo Err = {INVALID_HANDLE_VALUE, '\0'};
/*
* exitcodes: 0 == no, 1 == yes, 2 == error
*/
int
main(
int argc,
char *argv[])
{
char msg[300];
DWORD dwWritten;
int chars;
/*
* Make sure children (cl.exe and link.exe) are kept quiet.
*/
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX);
/*
* Make sure the compiler and linker aren't effected by the outside world.
*/
SetEnvironmentVariable("CL", "");
SetEnvironmentVariable("LINK", "");
if (argc > 1 && *argv[1] == '-') {
switch (*(argv[1]+1)) {
case 'c':
if (argc != 3) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -c <compiler option>\n"
"Tests for whether cl.exe supports an option\n"
"exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 2;
}
return CheckForCompilerFeature(argv[2]);
case 'l':
if (argc != 3) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -l <linker option>\n"
"Tests for whether link.exe supports an option\n"
"exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 2;
}
return CheckForLinkerFeature(argv[2]);
case 'f':
if (argc == 2) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -f <string> <substring>\n"
"Find a substring within another\n"
"exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 2;
} else if (argc == 3) {
/*
* If the string is blank, there is no match.
*/
return 0;
} else {
return IsIn(argv[2], argv[3]);
}
case 's':
if (argc == 2) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -s <substitutions file> <file>\n"
"Perform a set of string map type substutitions on a file\n"
"exitcodes: 0\n",
argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 2;
}
return SubstituteFile(argv[2], argv[3]);
case 'V':
if (argc != 4) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -V filename matchstring\n"
"Extract a version from a file:\n"
"eg: pkgIndex.tcl \"package ifneeded http\"",
argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 0;
}
printf("%s\n", GetVersionFromFile(argv[2], argv[3]));
return 0;
case 'Q':
if (argc != 3) {
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -Q path\n"
"Emit the fully qualified path\n"
"exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars,
&dwWritten, NULL);
return 2;
}
return QualifyPath(argv[2]);
}
}
chars = snprintf(msg, sizeof(msg) - 1,
"usage: %s -c|-f|-l|-Q|-s|-V ...\n"
"This is a little helper app to equalize shell differences between WinNT and\n"
"Win9x and get nmake.exe to accomplish its job.\n",
argv[0]);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, &dwWritten, NULL);
return 2;
}
static int
CheckForCompilerFeature(
const char *option)
{
STARTUPINFO si;
PROCESS_INFORMATION pi;
SECURITY_ATTRIBUTES sa;
DWORD threadID;
char msg[300];
BOOL ok;
HANDLE hProcess, h, pipeThreads[2];
char cmdline[100];
hProcess = GetCurrentProcess();
ZeroMemory(&pi, sizeof(PROCESS_INFORMATION));
ZeroMemory(&si, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdInput = INVALID_HANDLE_VALUE;
ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES));
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.lpSecurityDescriptor = NULL;
sa.bInheritHandle = FALSE;
/*
* Create a non-inheritible pipe.
*/
CreatePipe(&Out.pipe, &h, &sa, 0);
/*
* Dupe the write side, make it inheritible, and close the original.
*/
DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE,
DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE);
/*
* Same as above, but for the error side.
*/
CreatePipe(&Err.pipe, &h, &sa, 0);
DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE,
DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE);
/*
* Base command line.
*/
lstrcpy(cmdline, "cl.exe -nologo -c -TC -Zs -X -Fp.\\_junk.pch ");
/*
* Append our option for testing
*/
lstrcat(cmdline, option);
/*
* Filename to compile, which exists, but is nothing and empty.
*/
lstrcat(cmdline, " .\\nul");
ok = CreateProcess(
NULL, /* Module name. */
cmdline, /* Command line. */
NULL, /* Process handle not inheritable. */
NULL, /* Thread handle not inheritable. */
TRUE, /* yes, inherit handles. */
DETACHED_PROCESS, /* No console for you. */
NULL, /* Use parent's environment block. */
NULL, /* Use parent's starting directory. */
&si, /* Pointer to STARTUPINFO structure. */
&pi); /* Pointer to PROCESS_INFORMATION structure. */
if (!ok) {
DWORD err = GetLastError();
int chars = snprintf(msg, sizeof(msg) - 1,
"Tried to launch: \"%s\", but got error [%u]: ", cmdline, err);
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS|
FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPVOID)&msg[chars],
(300-chars), 0);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL);
return 2;
}
/*
* Close our references to the write handles that have now been inherited.
*/
CloseHandle(si.hStdOutput);
CloseHandle(si.hStdError);
WaitForInputIdle(pi.hProcess, 5000);
CloseHandle(pi.hThread);
/*
* Start the pipe reader threads.
*/
pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID);
pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID);
/*
* Block waiting for the process to end.
*/
WaitForSingleObject(pi.hProcess, INFINITE);
CloseHandle(pi.hProcess);
/*
* Wait for our pipe to get done reading, should it be a little slow.
*/
WaitForMultipleObjects(2, pipeThreads, TRUE, 500);
CloseHandle(pipeThreads[0]);
CloseHandle(pipeThreads[1]);
/*
* Look for the commandline warning code in both streams.
* - in MSVC 6 & 7 we get D4002, in MSVC 8 we get D9002.
*/
return !(strstr(Out.buffer, "D4002") != NULL
|| strstr(Err.buffer, "D4002") != NULL
|| strstr(Out.buffer, "D9002") != NULL
|| strstr(Err.buffer, "D9002") != NULL
|| strstr(Out.buffer, "D2021") != NULL
|| strstr(Err.buffer, "D2021") != NULL);
}
static int
CheckForLinkerFeature(
const char *option)
{
STARTUPINFO si;
PROCESS_INFORMATION pi;
SECURITY_ATTRIBUTES sa;
DWORD threadID;
char msg[300];
BOOL ok;
HANDLE hProcess, h, pipeThreads[2];
char cmdline[100];
hProcess = GetCurrentProcess();
ZeroMemory(&pi, sizeof(PROCESS_INFORMATION));
ZeroMemory(&si, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdInput = INVALID_HANDLE_VALUE;
ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES));
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.lpSecurityDescriptor = NULL;
sa.bInheritHandle = TRUE;
/*
* Create a non-inheritible pipe.
*/
CreatePipe(&Out.pipe, &h, &sa, 0);
/*
* Dupe the write side, make it inheritible, and close the original.
*/
DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE,
DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE);
/*
* Same as above, but for the error side.
*/
CreatePipe(&Err.pipe, &h, &sa, 0);
DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE,
DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE);
/*
* Base command line.
*/
lstrcpy(cmdline, "link.exe -nologo ");
/*
* Append our option for testing.
*/
lstrcat(cmdline, option);
ok = CreateProcess(
NULL, /* Module name. */
cmdline, /* Command line. */
NULL, /* Process handle not inheritable. */
NULL, /* Thread handle not inheritable. */
TRUE, /* yes, inherit handles. */
DETACHED_PROCESS, /* No console for you. */
NULL, /* Use parent's environment block. */
NULL, /* Use parent's starting directory. */
&si, /* Pointer to STARTUPINFO structure. */
&pi); /* Pointer to PROCESS_INFORMATION structure. */
if (!ok) {
DWORD err = GetLastError();
int chars = snprintf(msg, sizeof(msg) - 1,
"Tried to launch: \"%s\", but got error [%u]: ", cmdline, err);
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS|
FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPVOID)&msg[chars],
(300-chars), 0);
WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL);
return 2;
}
/*
* Close our references to the write handles that have now been inherited.
*/
CloseHandle(si.hStdOutput);
CloseHandle(si.hStdError);
WaitForInputIdle(pi.hProcess, 5000);
CloseHandle(pi.hThread);
/*
* Start the pipe reader threads.
*/
pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID);
pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID);
/*
* Block waiting for the process to end.
*/
WaitForSingleObject(pi.hProcess, INFINITE);
CloseHandle(pi.hProcess);
/*
* Wait for our pipe to get done reading, should it be a little slow.
*/
WaitForMultipleObjects(2, pipeThreads, TRUE, 500);
CloseHandle(pipeThreads[0]);
CloseHandle(pipeThreads[1]);
/*
* Look for the commandline warning code in the stderr stream.
*/
return !(strstr(Out.buffer, "LNK1117") != NULL ||
strstr(Err.buffer, "LNK1117") != NULL ||
strstr(Out.buffer, "LNK4044") != NULL ||
strstr(Err.buffer, "LNK4044") != NULL);
}
static DWORD WINAPI
ReadFromPipe(
LPVOID args)
{
pipeinfo *pi = (pipeinfo *) args;
char *lastBuf = pi->buffer;
DWORD dwRead;
BOOL ok;
again:
if (lastBuf - pi->buffer + CHUNK > STATICBUFFERSIZE) {
CloseHandle(pi->pipe);
return (DWORD)-1;
}
ok = ReadFile(pi->pipe, lastBuf, CHUNK, &dwRead, 0L);
if (!ok || dwRead == 0) {
CloseHandle(pi->pipe);
return 0;
}
lastBuf += dwRead;
goto again;
return 0; /* makes the compiler happy */
}
static int
IsIn(
const char *string,
const char *substring)
{
return (strstr(string, substring) != NULL);
}
/*
* GetVersionFromFile --
* Looks for a match string in a file and then returns the version
* following the match where a version is anything acceptable to
* package provide or package ifneeded.
*/
static const char *
GetVersionFromFile(
const char *filename,
const char *match)
{
size_t cbBuffer = 100;
static char szBuffer[100];
char *szResult = NULL;
FILE *fp = fopen(filename, "rt");
if (fp != NULL) {
/*
* Read data until we see our match string.
*/
while (fgets(szBuffer, cbBuffer, fp) != NULL) {
LPSTR p, q;
p = strstr(szBuffer, match);
if (p != NULL) {
/*
* Skip to first digit.
*/
while (*p && !isdigit(*p)) {
++p;
}
/*
* Find ending whitespace.
*/
q = p;
while (*q && (isalnum(*q) || *q == '.')) {
++q;
}
memcpy(szBuffer, p, q - p);
szBuffer[q-p] = 0;
szResult = szBuffer;
break;
}
}
fclose(fp);
}
return szResult;
}
/*
* List helpers for the SubstituteFile function
*/
typedef struct list_item_t {
struct list_item_t *nextPtr;
char * key;
char * value;
} list_item_t;
/* insert a list item into the list (list may be null) */
static list_item_t *
list_insert(list_item_t **listPtrPtr, const char *key, const char *value)
{
list_item_t *itemPtr = malloc(sizeof(list_item_t));
if (itemPtr) {
itemPtr->key = strdup(key);
itemPtr->value = strdup(value);
itemPtr->nextPtr = NULL;
while(*listPtrPtr) {
listPtrPtr = &(*listPtrPtr)->nextPtr;
}
*listPtrPtr = itemPtr;
}
return itemPtr;
}
static void
list_free(list_item_t **listPtrPtr)
{
list_item_t *tmpPtr, *listPtr = *listPtrPtr;
while (listPtr) {
tmpPtr = listPtr;
listPtr = listPtr->nextPtr;
free(tmpPtr->key);
free(tmpPtr->value);
free(tmpPtr);
}
}
/*
* SubstituteFile --
* As windows doesn't provide anything useful like sed and it's unreliable
* to use the tclsh you are building against (consider x-platform builds -
* eg compiling AMD64 target from IX86) we provide a simple substitution
* option here to handle autoconf style substitutions.
* The substitution file is whitespace and line delimited. The file should
* consist of lines matching the regular expression:
* \s*\S+\s+\S*$
*
* Usage is something like:
* nmakehlp -S << $** > $@
* @PACKAGE_NAME@ $(PACKAGE_NAME)
* @PACKAGE_VERSION@ $(PACKAGE_VERSION)
* <<
*/
static int
SubstituteFile(
const char *substitutions,
const char *filename)
{
size_t cbBuffer = 1024;
static char szBuffer[1024], szCopy[1024];
char *szResult = NULL;
list_item_t *substPtr = NULL;
FILE *fp, *sp;
fp = fopen(filename, "rt");
if (fp != NULL) {
/*
* Build a list of substutitions from the first filename
*/
sp = fopen(substitutions, "rt");
if (sp != NULL) {
while (fgets(szBuffer, cbBuffer, sp) != NULL) {
unsigned char *ks, *ke, *vs, *ve;
ks = (unsigned char*)szBuffer;
while (ks && *ks && isspace(*ks)) ++ks;
ke = ks;
while (ke && *ke && !isspace(*ke)) ++ke;
vs = ke;
while (vs && *vs && isspace(*vs)) ++vs;
ve = vs;
while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve;
*ke = 0, *ve = 0;
list_insert(&substPtr, (char*)ks, (char*)vs);
}
fclose(sp);
}
/* debug: dump the list */
#ifdef _DEBUG
{
int n = 0;
list_item_t *p = NULL;
for (p = substPtr; p != NULL; p = p->nextPtr, ++n) {
fprintf(stderr, "% 3d '%s' => '%s'\n", n, p->key, p->value);
}
}
#endif
/*
* Run the substitutions over each line of the input
*/
while (fgets(szBuffer, cbBuffer, fp) != NULL) {
list_item_t *p = NULL;
for (p = substPtr; p != NULL; p = p->nextPtr) {
char *m = strstr(szBuffer, p->key);
if (m) {
char *cp, *op, *sp;
cp = szCopy;
op = szBuffer;
while (op != m) *cp++ = *op++;
sp = p->value;
while (sp && *sp) *cp++ = *sp++;
op += strlen(p->key);
while (*op) *cp++ = *op++;
*cp = 0;
memcpy(szBuffer, szCopy, sizeof(szCopy));
}
}
printf(szBuffer);
}
list_free(&substPtr);
}
fclose(fp);
return 0;
}
/*
* QualifyPath --
*
* This composes the current working directory with a provided path
* and returns the fully qualified and normalized path.
* Mostly needed to setup paths for testing.
*/
static int
QualifyPath(
const char *szPath)
{
char szCwd[MAX_PATH + 1];
char szTmp[MAX_PATH + 1];
char *p;
GetCurrentDirectory(MAX_PATH, szCwd);
while ((p = strchr(szPath, '/')) && *p)
*p = '\\';
PathCombine(szTmp, szCwd, szPath);
PathCanonicalize(szCwd, szTmp);
printf("%s\n", szCwd);
return 0;
}
/*
* Local variables:
* mode: c
* c-basic-offset: 4
* fill-column: 78
* indent-tabs-mode: t
* tab-width: 8
* End:
*/
-711
View File
@@ -1,711 +0,0 @@
#------------------------------------------------------------------------------
# rules.vc --
#
# Microsoft Visual C++ makefile include for decoding the commandline
# macros. This file does not need editing to build Tcl.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# Copyright (c) 2001-2003 David Gravereaux.
# Copyright (c) 2003-2008 Patrick Thoyts
#------------------------------------------------------------------------------
!ifndef _RULES_VC
_RULES_VC = 1
cc32 = $(CC) # built-in default.
link32 = link
lib32 = lib
rc32 = $(RC) # built-in default.
!ifndef INSTALLDIR
### Assume the normal default.
_INSTALLDIR = C:\Program Files\Tcl
!else
### Fix the path separators.
_INSTALLDIR = $(INSTALLDIR:/=\)
!endif
#----------------------------------------------------------
# Set the proper copy method to avoid overwrite questions
# to the user when copying files and selecting the right
# "delete all" method.
#----------------------------------------------------------
!if "$(OS)" == "Windows_NT"
RMDIR = rmdir /S /Q
ERRNULL = 2>NUL
!if ![ver | find "4.0" > nul]
CPY = echo y | xcopy /i >NUL
COPY = copy >NUL
!else
CPY = xcopy /i /y >NUL
COPY = copy /y >NUL
!endif
!else # "$(OS)" != "Windows_NT"
CPY = xcopy /i >_JUNK.OUT # On Win98 NUL does not work here.
COPY = copy >_JUNK.OUT # On Win98 NUL does not work here.
RMDIR = deltree /Y
NULL = \NUL # Used in testing directory existence
ERRNULL = >NUL # Win9x shell cannot redirect stderr
!endif
MKDIR = mkdir
#------------------------------------------------------------------------------
# Determine the host and target architectures and compiler version.
#------------------------------------------------------------------------------
_HASH=^#
_VC_MANIFEST_EMBED_EXE=
_VC_MANIFEST_EMBED_DLL=
VCVER=0
!if ![echo VCVERSION=_MSC_VER > vercl.x] \
&& ![echo $(_HASH)if defined(_M_IX86) >> vercl.x] \
&& ![echo ARCH=IX86 >> vercl.x] \
&& ![echo $(_HASH)elif defined(_M_AMD64) >> vercl.x] \
&& ![echo ARCH=AMD64 >> vercl.x] \
&& ![echo $(_HASH)endif >> vercl.x] \
&& ![cl -nologo -TC -P vercl.x $(ERRNULL)]
!include vercl.i
!if ![echo VCVER= ^\> vercl.vc] \
&& ![set /a $(VCVERSION) / 100 - 6 >> vercl.vc]
!include vercl.vc
!endif
!endif
!if ![del $(ERRNUL) /q/f vercl.x vercl.i vercl.vc]
!endif
!if ![reg query HKLM\Hardware\Description\System\CentralProcessor\0 /v Identifier | findstr /i x86]
NATIVE_ARCH=IX86
!else
NATIVE_ARCH=AMD64
!endif
# Since MSVC8 we must deal with manifest resources.
!if $(VCVERSION) >= 1400
_VC_MANIFEST_EMBED_EXE=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;1
_VC_MANIFEST_EMBED_DLL=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;2
!endif
!ifndef MACHINE
MACHINE=$(ARCH)
!endif
!ifndef CFG_ENCODING
CFG_ENCODING = \"cp1252\"
!endif
!message ===============================================================================
#----------------------------------------------------------
# build the helper app we need to overcome nmake's limiting
# environment.
#----------------------------------------------------------
!if !exist(nmakehlp.exe)
!if [$(cc32) -nologo nmakehlp.c -link -subsystem:console > nul]
!endif
!endif
#----------------------------------------------------------
# Test for compiler features
#----------------------------------------------------------
### test for optimizations
!if [nmakehlp -c -Ot]
!message *** Compiler has 'Optimizations'
OPTIMIZING = 1
!else
!message *** Compiler does not have 'Optimizations'
OPTIMIZING = 0
!endif
OPTIMIZATIONS =
!if [nmakehlp -c -Ot]
OPTIMIZATIONS = $(OPTIMIZATIONS) -Ot
!endif
!if [nmakehlp -c -Oi]
OPTIMIZATIONS = $(OPTIMIZATIONS) -Oi
!endif
!if [nmakehlp -c -Op]
OPTIMIZATIONS = $(OPTIMIZATIONS) -Op
!endif
!if [nmakehlp -c -fp:strict]
OPTIMIZATIONS = $(OPTIMIZATIONS) -fp:strict
!endif
!if [nmakehlp -c -Gs]
OPTIMIZATIONS = $(OPTIMIZATIONS) -Gs
!endif
!if [nmakehlp -c -GS]
OPTIMIZATIONS = $(OPTIMIZATIONS) -GS
!endif
!if [nmakehlp -c -GL]
OPTIMIZATIONS = $(OPTIMIZATIONS) -GL
!endif
DEBUGFLAGS =
!if [nmakehlp -c -RTC1]
DEBUGFLAGS = $(DEBUGFLAGS) -RTC1
!elseif [nmakehlp -c -GZ]
DEBUGFLAGS = $(DEBUGFLAGS) -GZ
!endif
COMPILERFLAGS =-W3 -DUNICODE -D_UNICODE
# In v13 -GL and -YX are incompatible.
!if [nmakehlp -c -YX]
!if ![nmakehlp -c -GL]
OPTIMIZATIONS = $(OPTIMIZATIONS) -YX
!endif
!endif
!if "$(MACHINE)" == "IX86"
### test for pentium errata
!if [nmakehlp -c -QI0f]
!message *** Compiler has 'Pentium 0x0f fix'
COMPILERFLAGS = $(COMPILERFLAGS) -QI0f
!else
!message *** Compiler does not have 'Pentium 0x0f fix'
!endif
!endif
!if "$(MACHINE)" == "IA64"
### test for Itanium errata
!if [nmakehlp -c -QIA64_Bx]
!message *** Compiler has 'B-stepping errata workarounds'
COMPILERFLAGS = $(COMPILERFLAGS) -QIA64_Bx
!else
!message *** Compiler does not have 'B-stepping errata workarounds'
!endif
!endif
!if "$(MACHINE)" == "IX86"
### test for -align:4096, when align:512 will do.
!if [nmakehlp -l -opt:nowin98]
!message *** Linker has 'Win98 alignment problem'
ALIGN98_HACK = 1
!else
!message *** Linker does not have 'Win98 alignment problem'
ALIGN98_HACK = 0
!endif
!else
ALIGN98_HACK = 0
!endif
LINKERFLAGS =
!if [nmakehlp -l -ltcg]
LINKERFLAGS =-ltcg
!endif
#----------------------------------------------------------
# Decode the options requested.
#----------------------------------------------------------
!if "$(OPTS)" == "" || [nmakehlp -f "$(OPTS)" "none"]
STATIC_BUILD = 0
TCL_THREADS = 1
DEBUG = 0
SYMBOLS = 0
PROFILE = 0
PGO = 0
MSVCRT = 0
LOIMPACT = 0
TCL_USE_STATIC_PACKAGES = 0
USE_THREAD_ALLOC = 1
UNCHECKED = 0
!else
!if [nmakehlp -f $(OPTS) "static"]
!message *** Doing static
STATIC_BUILD = 1
!else
STATIC_BUILD = 0
!endif
!if [nmakehlp -f $(OPTS) "msvcrt"]
!message *** Doing msvcrt
MSVCRT = 1
!else
MSVCRT = 0
!endif
!if [nmakehlp -f $(OPTS) "staticpkg"]
!message *** Doing staticpkg
TCL_USE_STATIC_PACKAGES = 1
!else
TCL_USE_STATIC_PACKAGES = 0
!endif
!if [nmakehlp -f $(OPTS) "nothreads"]
!message *** Compile explicitly for non-threaded tcl
TCL_THREADS = 0
!else
TCL_THREADS = 1
USE_THREAD_ALLOC= 1
!endif
!if [nmakehlp -f $(OPTS) "symbols"]
!message *** Doing symbols
DEBUG = 1
!else
DEBUG = 0
!endif
!if [nmakehlp -f $(OPTS) "pdbs"]
!message *** Doing pdbs
SYMBOLS = 1
!else
SYMBOLS = 0
!endif
!if [nmakehlp -f $(OPTS) "profile"]
!message *** Doing profile
PROFILE = 1
!else
PROFILE = 0
!endif
!if [nmakehlp -f $(OPTS) "pgi"]
!message *** Doing profile guided optimization instrumentation
PGO = 1
!elseif [nmakehlp -f $(OPTS) "pgo"]
!message *** Doing profile guided optimization
PGO = 2
!else
PGO = 0
!endif
!if [nmakehlp -f $(OPTS) "loimpact"]
!message *** Doing loimpact
LOIMPACT = 1
!else
LOIMPACT = 0
!endif
!if [nmakehlp -f $(OPTS) "thrdalloc"]
!message *** Doing thrdalloc
USE_THREAD_ALLOC = 1
!endif
!if [nmakehlp -f $(OPTS) "tclalloc"]
!message *** Doing tclalloc
USE_THREAD_ALLOC = 0
!endif
!if [nmakehlp -f $(OPTS) "unchecked"]
!message *** Doing unchecked
UNCHECKED = 1
!else
UNCHECKED = 0
!endif
!endif
!if !$(STATIC_BUILD)
# Make sure we don't build overly fat DLLs.
MSVCRT = 1
# We shouldn't statically put the extensions inside the shell when dynamic.
TCL_USE_STATIC_PACKAGES = 0
!endif
#----------------------------------------------------------
# Figure-out how to name our intermediate and output directories.
# We wouldn't want different builds to use the same .obj files
# by accident.
#----------------------------------------------------------
#----------------------------------------
# Naming convention:
# t = full thread support.
# s = static library (as opposed to an
# import library)
# g = linked to the debug enabled C
# run-time.
# x = special static build when it
# links to the dynamic C run-time.
#----------------------------------------
SUFX = tsgx
!if $(DEBUG)
BUILDDIRTOP = Debug
!else
BUILDDIRTOP = Release
!endif
!if "$(MACHINE)" != "IX86"
BUILDDIRTOP =$(BUILDDIRTOP)_$(MACHINE)
!endif
!if $(VCVER) > 6
BUILDDIRTOP =$(BUILDDIRTOP)_VC$(VCVER)
!endif
!if !$(DEBUG) || $(DEBUG) && $(UNCHECKED)
SUFX = $(SUFX:g=)
!endif
TMP_DIRFULL = .\$(BUILDDIRTOP)\$(PROJECT)_ThreadedDynamicStaticX
!if !$(STATIC_BUILD)
TMP_DIRFULL = $(TMP_DIRFULL:Static=)
SUFX = $(SUFX:s=)
EXT = dll
!if $(MSVCRT)
TMP_DIRFULL = $(TMP_DIRFULL:X=)
SUFX = $(SUFX:x=)
!endif
!else
TMP_DIRFULL = $(TMP_DIRFULL:Dynamic=)
EXT = lib
!if !$(MSVCRT)
TMP_DIRFULL = $(TMP_DIRFULL:X=)
SUFX = $(SUFX:x=)
!endif
!endif
!if !$(TCL_THREADS)
TMP_DIRFULL = $(TMP_DIRFULL:Threaded=)
SUFX = $(SUFX:t=)
!endif
!ifndef TMP_DIR
TMP_DIR = $(TMP_DIRFULL)
!ifndef OUT_DIR
OUT_DIR = .\$(BUILDDIRTOP)
!endif
!else
!ifndef OUT_DIR
OUT_DIR = $(TMP_DIR)
!endif
!endif
#----------------------------------------------------------
# Decode the statistics requested.
#----------------------------------------------------------
!if "$(STATS)" == "" || [nmakehlp -f "$(STATS)" "none"]
TCL_MEM_DEBUG = 0
TCL_COMPILE_DEBUG = 0
!else
!if [nmakehlp -f $(STATS) "memdbg"]
!message *** Doing memdbg
TCL_MEM_DEBUG = 1
!else
TCL_MEM_DEBUG = 0
!endif
!if [nmakehlp -f $(STATS) "compdbg"]
!message *** Doing compdbg
TCL_COMPILE_DEBUG = 1
!else
TCL_COMPILE_DEBUG = 0
!endif
!endif
#----------------------------------------------------------
# Decode the checks requested.
#----------------------------------------------------------
!if "$(CHECKS)" == "" || [nmakehlp -f "$(CHECKS)" "none"]
TCL_NO_DEPRECATED = 0
WARNINGS = -W3
!else
!if [nmakehlp -f $(CHECKS) "nodep"]
!message *** Doing nodep check
TCL_NO_DEPRECATED = 1
!else
TCL_NO_DEPRECATED = 0
!endif
!if [nmakehlp -f $(CHECKS) "fullwarn"]
!message *** Doing full warnings check
WARNINGS = -W4
!if [nmakehlp -l -warn:3]
LINKERFLAGS = $(LINKERFLAGS) -warn:3
!endif
!else
WARNINGS = -W3
!endif
!if [nmakehlp -f $(CHECKS) "64bit"] && [nmakehlp -c -Wp64]
!message *** Doing 64bit portability warnings
WARNINGS = $(WARNINGS) -Wp64
!endif
!endif
!if $(PGO) > 1
!if [nmakehlp -l -ltcg:pgoptimize]
LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pgoptimize
!else
MSG=^
This compiler does not support profile guided optimization.
!error $(MSG)
!endif
!elseif $(PGO) > 0
!if [nmakehlp -l -ltcg:pginstrument]
LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pginstrument
!else
MSG=^
This compiler does not support profile guided optimization.
!error $(MSG)
!endif
!endif
#----------------------------------------------------------
# Set our defines now armed with our options.
#----------------------------------------------------------
OPTDEFINES = -DTCL_CFGVAL_ENCODING=$(CFG_ENCODING) -DSTDC_HEADERS
!if $(TCL_MEM_DEBUG)
OPTDEFINES = $(OPTDEFINES) -DTCL_MEM_DEBUG
!endif
!if $(TCL_COMPILE_DEBUG)
OPTDEFINES = $(OPTDEFINES) -DTCL_COMPILE_DEBUG -DTCL_COMPILE_STATS
!endif
!if $(TCL_THREADS)
OPTDEFINES = $(OPTDEFINES) -DTCL_THREADS=1
!if $(USE_THREAD_ALLOC)
OPTDEFINES = $(OPTDEFINES) -DUSE_THREAD_ALLOC=1
!endif
!endif
!if $(STATIC_BUILD)
OPTDEFINES = $(OPTDEFINES) -DSTATIC_BUILD
!endif
!if $(TCL_NO_DEPRECATED)
OPTDEFINES = $(OPTDEFINES) -DTCL_NO_DEPRECATED
!endif
!if !$(DEBUG)
OPTDEFINES = $(OPTDEFINES) -DNDEBUG
!if $(OPTIMIZING)
OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_OPTIMIZED
!endif
!endif
!if $(PROFILE)
OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_PROFILED
!endif
!if "$(MACHINE)" == "IA64" || "$(MACHINE)" == "AMD64"
OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_DO64BIT
!endif
!if $(VCVERSION) < 1300
OPTDEFINES = $(OPTDEFINES) -DNO_STRTOI64
!endif
#----------------------------------------------------------
# Locate the Tcl headers to build against
#----------------------------------------------------------
!if "$(PROJECT)" == "tcl"
_TCL_H = ..\generic\tcl.h
!else
# If INSTALLDIR set to tcl root dir then reset to the lib dir.
!if exist("$(_INSTALLDIR)\include\tcl.h")
_INSTALLDIR=$(_INSTALLDIR)\lib
!endif
!if !defined(TCLDIR)
!if exist("$(_INSTALLDIR)\..\include\tcl.h")
TCLINSTALL = 1
_TCLDIR = $(_INSTALLDIR)\..
_TCL_H = $(_INSTALLDIR)\..\include\tcl.h
TCLDIR = $(_INSTALLDIR)\..
!else
MSG=^
Failed to find tcl.h. Set the TCLDIR macro.
!error $(MSG)
!endif
!else
_TCLDIR = $(TCLDIR:/=\)
!if exist("$(_TCLDIR)\include\tcl.h")
TCLINSTALL = 1
_TCL_H = $(_TCLDIR)\include\tcl.h
!elseif exist("$(_TCLDIR)\generic\tcl.h")
TCLINSTALL = 0
_TCL_H = $(_TCLDIR)\generic\tcl.h
!else
MSG =^
Failed to find tcl.h. The TCLDIR macro does not appear correct.
!error $(MSG)
!endif
!endif
!endif
#--------------------------------------------------------------
# Extract various version numbers from tcl headers
# The generated file is then included in the makefile.
#--------------------------------------------------------------
!if [echo REM = This file is generated from rules.vc > versions.vc]
!endif
!if [echo TCL_MAJOR_VERSION = \>> versions.vc] \
&& [nmakehlp -V "$(_TCL_H)" TCL_MAJOR_VERSION >> versions.vc]
!endif
!if [echo TCL_MINOR_VERSION = \>> versions.vc] \
&& [nmakehlp -V "$(_TCL_H)" TCL_MINOR_VERSION >> versions.vc]
!endif
!if [echo TCL_PATCH_LEVEL = \>> versions.vc] \
&& [nmakehlp -V "$(_TCL_H)" TCL_PATCH_LEVEL >> versions.vc]
!endif
# If building the tcl core then we need additional package versions
!if "$(PROJECT)" == "tcl"
!if [echo PKG_HTTP_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\http\pkgIndex.tcl http >> versions.vc]
!endif
!if [echo PKG_TCLTEST_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\tcltest\pkgIndex.tcl tcltest >> versions.vc]
!endif
!if [echo PKG_MSGCAT_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\msgcat\pkgIndex.tcl msgcat >> versions.vc]
!endif
!if [echo PKG_PLATFORM_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\platform\pkgIndex.tcl "platform " >> versions.vc]
!endif
!if [echo PKG_SHELL_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\platform\pkgIndex.tcl "platform::shell" >> versions.vc]
!endif
!if [echo PKG_DDE_VER = \>> versions.vc] \
&& [nmakehlp -V ..\library\dde\pkgIndex.tcl "dde " >> versions.vc]
!endif
!if [echo PKG_REG_VER =\>> versions.vc] \
&& [nmakehlp -V ..\library\reg\pkgIndex.tcl registry >> versions.vc]
!endif
!endif
!include versions.vc
#--------------------------------------------------------------
# Setup tcl version dependent stuff headers
#--------------------------------------------------------------
!if "$(PROJECT)" != "tcl"
TCL_VERSION = $(TCL_MAJOR_VERSION)$(TCL_MINOR_VERSION)
!if $(TCL_VERSION) < 81
TCL_DOES_STUBS = 0
!else
TCL_DOES_STUBS = 1
!endif
!if $(TCLINSTALL)
TCLSH = "$(_TCLDIR)\bin\tclsh$(TCL_VERSION)$(SUFX).exe"
!if !exist($(TCLSH)) && $(TCL_THREADS)
TCLSH = "$(_TCLDIR)\bin\tclsh$(TCL_VERSION)t$(SUFX).exe"
!endif
TCLSTUBLIB = "$(_TCLDIR)\lib\tclstub$(TCL_VERSION).lib"
TCLIMPLIB = "$(_TCLDIR)\lib\tcl$(TCL_VERSION)$(SUFX).lib"
TCL_LIBRARY = $(_TCLDIR)\lib
TCLREGLIB = "$(_TCLDIR)\lib\tclreg13$(SUFX:t=).lib"
TCLDDELIB = "$(_TCLDIR)\lib\tcldde14$(SUFX:t=).lib"
COFFBASE = \must\have\tcl\sources\to\build\this\target
TCLTOOLSDIR = \must\have\tcl\sources\to\build\this\target
TCL_INCLUDES = -I"$(_TCLDIR)\include"
!else
TCLSH = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)$(SUFX).exe"
!if !exist($(TCLSH)) && $(TCL_THREADS)
TCLSH = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)t$(SUFX).exe"
!endif
TCLSTUBLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclstub$(TCL_VERSION).lib"
TCLIMPLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tcl$(TCL_VERSION)$(SUFX).lib"
TCL_LIBRARY = $(_TCLDIR)\library
TCLREGLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclreg13$(SUFX:t=).lib"
TCLDDELIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tcldde14$(SUFX:t=).lib"
COFFBASE = "$(_TCLDIR)\win\coffbase.txt"
TCLTOOLSDIR = $(_TCLDIR)\tools
TCL_INCLUDES = -I"$(_TCLDIR)\generic" -I"$(_TCLDIR)\win"
!endif
!endif
#-------------------------------------------------------------------------
# Locate the Tk headers to build against
#-------------------------------------------------------------------------
!if "$(PROJECT)" == "tk"
_TK_H = ..\generic\tk.h
_INSTALLDIR = $(_INSTALLDIR)\..
!endif
!ifdef PROJECT_REQUIRES_TK
!if !defined(TKDIR)
!if exist("$(_INSTALLDIR)\..\include\tk.h")
TKINSTALL = 1
_TKDIR = $(_INSTALLDIR)\..
_TK_H = $(_TKDIR)\include\tk.h
TKDIR = $(_TKDIR)
!elseif exist("$(_TCLDIR)\include\tk.h")
TKINSTALL = 1
_TKDIR = $(_TCLDIR)
_TK_H = $(_TKDIR)\include\tk.h
TKDIR = $(_TKDIR)
!endif
!else
_TKDIR = $(TKDIR:/=\)
!if exist("$(_TKDIR)\include\tk.h")
TKINSTALL = 1
_TK_H = $(_TKDIR)\include\tk.h
!elseif exist("$(_TKDIR)\generic\tk.h")
TKINSTALL = 0
_TK_H = $(_TKDIR)\generic\tk.h
!else
MSG =^
Failed to find tk.h. The TKDIR macro does not appear correct.
!error $(MSG)
!endif
!endif
!endif
#-------------------------------------------------------------------------
# Extract Tk version numbers
#-------------------------------------------------------------------------
!if defined(PROJECT_REQUIRES_TK) || "$(PROJECT)" == "tk"
!if [echo TK_MAJOR_VERSION = \>> versions.vc] \
&& [nmakehlp -V $(_TK_H) TK_MAJOR_VERSION >> versions.vc]
!endif
!if [echo TK_MINOR_VERSION = \>> versions.vc] \
&& [nmakehlp -V $(_TK_H) TK_MINOR_VERSION >> versions.vc]
!endif
!if [echo TK_PATCH_LEVEL = \>> versions.vc] \
&& [nmakehlp -V $(_TK_H) TK_PATCH_LEVEL >> versions.vc]
!endif
!include versions.vc
TK_DOTVERSION = $(TK_MAJOR_VERSION).$(TK_MINOR_VERSION)
TK_VERSION = $(TK_MAJOR_VERSION)$(TK_MINOR_VERSION)
!if "$(PROJECT)" != "tk"
!if $(TKINSTALL)
WISH = "$(_TKDIR)\bin\wish$(TK_VERSION)$(SUFX).exe"
TKSTUBLIB = "$(_TKDIR)\lib\tkstub$(TK_VERSION).lib"
TKIMPLIB = "$(_TKDIR)\lib\tk$(TK_VERSION)$(SUFX).lib"
TK_INCLUDES = -I"$(_TKDIR)\include"
!else
WISH = "$(_TKDIR)\win\$(BUILDDIRTOP)\wish$(TCL_VERSION)$(SUFX).exe"
TKSTUBLIB = "$(_TKDIR)\win\$(BUILDDIRTOP)\tkstub$(TCL_VERSION).lib"
TKIMPLIB = "$(_TKDIR)\win\$(BUILDDIRTOP)\tk$(TCL_VERSION)$(SUFX).lib"
TK_INCLUDES = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib"
!endif
!endif
!endif
#----------------------------------------------------------
# Display stats being used.
#----------------------------------------------------------
!message *** Intermediate directory will be '$(TMP_DIR)'
!message *** Output directory will be '$(OUT_DIR)'
!message *** Suffix for binaries will be '$(SUFX)'
!message *** Optional defines are '$(OPTDEFINES)'
!message *** Compiler version $(VCVER). Target machine is $(MACHINE)
!message *** Host architecture is $(NATIVE_ARCH)
!message *** Compiler options '$(COMPILERFLAGS) $(OPTIMIZATIONS) $(DEBUGFLAGS) $(WARNINGS)'
!message *** Link options '$(LINKERFLAGS)'
!endif
-18
View File
@@ -27,9 +27,6 @@
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if you have the `isnan' function. */
#undef HAVE_ISNAN
/* Define to 1 if you have the `localtime_r' function. */
#undef HAVE_LOCALTIME_R
@@ -45,27 +42,12 @@
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the pread() function. */
#undef HAVE_PREAD
/* Define to 1 if you have the pread64() function. */
#undef HAVE_PREAD64
/* Define to 1 if you have the pwrite() function. */
#undef HAVE_PWRITE
/* Define to 1 if you have the pwrite64() function. */
#undef HAVE_PWRITE64
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the strchrnul() function */
#undef HAVE_STRCHRNUL
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
Vendored
+4566 -2733
View File
File diff suppressed because it is too large Load Diff
+93 -147
View File
@@ -69,6 +69,19 @@
# The filename extension for executables on the
# target platform. "" for Unix and ".exe" for windows.
#
# The generated configure script will make an attempt to guess
# at all of the above parameters. You can override any of
# the guesses by setting the environment variable named
# "config_AAAA" where "AAAA" is the name of the parameter
# described above. (Exception: srcdir cannot be set this way.)
# If you have a file that sets one or more of these environment
# variables, you can invoke configure as follows:
#
# configure --with-hints=FILE
#
# where FILE is the name of the file that sets the environment
# variables. FILE should be an absolute pathname.
#
# This configure.in file is easy to reuse on other projects. Just
# change the argument to AC_INIT(). And disable any features that
# you don't need (for example BLT) by erasing or commenting out
@@ -85,11 +98,17 @@ AC_MSG_ERROR([configure script is out of date:
please regen with autoconf])
fi
dnl Put the RCS revision string after AC_INIT so that it will also
dnl show in in configure.
# The following RCS revision string applies to configure.in
# $Revision: 1.56 $
#########
# Programs needed
#
AC_PROG_LIBTOOL
AC_PROG_INSTALL
AC_PROG_AWK
#########
# Enable large file support (if special flags are necessary)
@@ -108,7 +127,7 @@ AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h])
#########
# Figure out whether or not we have these functions
#
AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64])
AC_CHECK_FUNCS([usleep fdatasync localtime_r gmtime_r localtime_s utime malloc_usable_size])
#########
# By default, we use the amalgamation (this may be changed below...)
@@ -120,7 +139,7 @@ USE_AMALGAMATION=1
# if not, then we fall back to plain tclsh.
# TODO: try other versions before falling back?
#
AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.5 tclsh], none)
if test "$TCLSH_CMD" = "none"; then
# If we can't find a local tclsh, then building the amalgamation will fail.
# We act as though --disable-amalgamation has been used.
@@ -161,6 +180,41 @@ VERSION_NUMBER=[`cat $srcdir/VERSION \
AC_MSG_NOTICE(Version number set to $VERSION_NUMBER)
AC_SUBST(VERSION_NUMBER)
#########
# Check to see if the --with-hints=FILE option is used. If there is none,
# then check for a files named "$host.hints" and ../$hosts.hints where
# $host is the hostname of the build system. If still no hints are
# found, try looking in $system.hints and ../$system.hints where
# $system is the result of uname -s.
#
AC_ARG_WITH(hints,
AC_HELP_STRING([--with-hints=FILE],[Read configuration options from FILE]),
hints=$withval)
if test "$hints" = ""; then
host=`hostname | sed 's/\..*//'`
if test -r $host.hints; then
hints=$host.hints
else
if test -r ../$host.hints; then
hints=../$host.hints
fi
fi
fi
if test "$hints" = ""; then
sys=`uname -s`
if test -r $sys.hints; then
hints=$sys.hints
else
if test -r ../$sys.hints; then
hints=../$sys.hints
fi
fi
fi
if test "$hints" != ""; then
AC_MSG_RESULT(reading hints from $hints)
. $hints
fi
#########
# Locate a compiler for the build machine. This compiler should
# generate command-line programs that run on the build machine.
@@ -182,7 +236,7 @@ AC_SUBST(BUILD_CC)
# Do we want to support multithreaded use of sqlite
#
AC_ARG_ENABLE(threadsafe,
AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
AC_HELP_STRING([--enable-threadsafe],[Support threadsafe operation]),,enable_threadsafe=yes)
AC_MSG_CHECKING([whether to support threadsafe operation])
if test "$enable_threadsafe" = "no"; then
SQLITE_THREADSAFE=0
@@ -195,9 +249,25 @@ AC_SUBST(SQLITE_THREADSAFE)
if test "$SQLITE_THREADSAFE" = "1"; then
AC_SEARCH_LIBS(pthread_create, pthread)
AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
fi
##########
# Do we want to allow a connection created in one thread to be used
# in another thread. This does not work on many Linux systems (ex: RedHat 9)
# due to bugs in the threading implementations. This is thus off by default.
#
AC_ARG_ENABLE(cross-thread-connections,
AC_HELP_STRING([--enable-cross-thread-connections],[Allow connection sharing across threads]),,enable_xthreadconnect=no)
AC_MSG_CHECKING([whether to allow connections to be shared across threads])
if test "$enable_xthreadconnect" = "no"; then
XTHREADCONNECT=''
AC_MSG_RESULT([no])
else
XTHREADCONNECT='-DSQLITE_ALLOW_XTHREAD_CONNECT=1'
AC_MSG_RESULT([yes])
fi
AC_SUBST(XTHREADCONNECT)
##########
# Do we want to support release
#
@@ -270,18 +340,28 @@ else
TARGET_EXEEXT=$config_TARGET_EXEEXT
fi
if test "$TARGET_EXEEXT" = ".exe"; then
SQLITE_OS_UNIX=0
SQLITE_OS_WIN=1
CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1"
if test $OS2_SHELL ; then
SQLITE_OS_UNIX=0
SQLITE_OS_WIN=0
SQLITE_OS_OS2=1
CFLAGS="$CFLAGS -DSQLITE_OS_OS2=1"
else
SQLITE_OS_UNIX=0
SQLITE_OS_WIN=1
SQLITE_OS_OS2=0
CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1"
fi
else
SQLITE_OS_UNIX=1
SQLITE_OS_WIN=0
SQLITE_OS_OS2=0
CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1"
fi
AC_SUBST(BUILD_EXEEXT)
AC_SUBST(SQLITE_OS_UNIX)
AC_SUBST(SQLITE_OS_WIN)
AC_SUBST(SQLITE_OS_OS2)
AC_SUBST(TARGET_EXEEXT)
##########
@@ -320,34 +400,6 @@ if test "${use_tcl}" = "yes" ; then
fi
fi
# On ubuntu 14.10, $auto_path on tclsh is not quite correct.
# So try again after applying corrections.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# Recent versions of Xcode on Macs hid the tclConfig.sh file
# in a strange place.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
@@ -458,7 +510,6 @@ if test "${use_tcl}" = "yes" ; then
AC_SUBST(TCL_STUB_LIB_FILE)
AC_SUBST(TCL_STUB_LIB_FLAG)
AC_SUBST(TCL_STUB_LIB_SPEC)
AC_SUBST(TCL_SHLIB_SUFFIX)
fi
fi
if test "${use_tcl}" = "no" ; then
@@ -475,24 +526,11 @@ AC_SUBST(HAVE_TCL)
TARGET_READLINE_LIBS=""
TARGET_READLINE_INC=""
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
AC_ARG_ENABLE([editline],
[AC_HELP_STRING([--enable-editline],[enable BSD editline support])],
[with_editline=$enableval],
[with_editline=auto])
AC_ARG_ENABLE([readline],
[AC_HELP_STRING([--disable-readline],[disable readline support])],
[AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])],
[with_readline=$enableval],
[with_readline=auto])
if test x"$with_editline" != xno; then
sLIBS=$LIBS
LIBS=""
TARGET_HAVE_EDITLINE=1
AC_SEARCH_LIBS(readline,edit,[with_readline=no],[TARGET_HAVE_EDITLINE=0])
TARGET_READLINE_LIBS=$LIBS
LIBS=$sLIBS
fi
if test x"$with_readline" != xno; then
found="yes"
@@ -547,7 +585,6 @@ fi
AC_SUBST(TARGET_READLINE_LIBS)
AC_SUBST(TARGET_READLINE_INC)
AC_SUBST(TARGET_HAVE_READLINE)
AC_SUBST(TARGET_HAVE_EDITLINE)
##########
# Figure out what C libraries are required to compile programs
@@ -560,7 +597,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
[use_debug=$enableval],[use_debug=no])
if test "${use_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
TARGET_DEBUG="-DSQLITE_DEBUG=1"
else
TARGET_DEBUG="-DNDEBUG"
fi
@@ -576,17 +613,11 @@ if test "${use_amalgamation}" != "yes" ; then
fi
AC_SUBST(USE_AMALGAMATION)
#########
# Look for zlib. Only needed by extensions and by the sqlite3.exe shell
AC_CHECK_HEADERS(zlib.h)
AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
AC_SUBST(HAVE_ZLIB)
#########
# See whether we should allow loadable extensions
AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
[Disable loading of external extensions]),
[use_loadextension=$enableval],[use_loadextension=yes])
AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--enable-load-extension],
[Enable loading of external extensions]),
[use_loadextension=$enableval],[use_loadextension=no])
if test "${use_loadextension}" = "yes" ; then
OPT_FEATURE_FLAGS=""
AC_SEARCH_LIBS(dlopen, dl)
@@ -594,93 +625,8 @@ else
OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
fi
##########
# Do we want to support memsys3 and/or memsys5
#
AC_ARG_ENABLE(memsys5,
AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
[enable_memsys5=yes],[enable_memsys5=no])
AC_MSG_CHECKING([whether to support MEMSYS5])
if test "${enable_memsys5}" = "yes"; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
fi
AC_ARG_ENABLE(memsys3,
AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
[enable_memsys3=yes],[enable_memsys3=no])
AC_MSG_CHECKING([whether to support MEMSYS3])
if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
fi
#########
# See whether we should enable Full Text Search extensions
AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
[Enable the FTS3 extension]),
[enable_fts3=yes],[enable_fts3=no])
if test "${enable_fts3}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
fi
AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
[Enable the FTS4 extension]),
[enable_fts4=yes],[enable_fts4=no])
if test "${enable_fts4}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
AC_SEARCH_LIBS([log],[m])
fi
AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
[Enable the FTS5 extension]),
[enable_fts5=yes],[enable_fts5=no])
if test "${enable_fts5}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
AC_SEARCH_LIBS([log],[m])
fi
#########
# See whether we should enable JSON1
AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
[Enable the JSON1 extension]),
[enable_json1=yes],[enable_json1=no])
if test "${enable_json1}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
fi
#########
# See whether we should enable the LIMIT clause on UPDATE and DELETE
# statements.
AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
[Enable the UPDATE/DELETE LIMIT clause]),
[enable_udlimit=yes],[enable_udlimit=no])
if test "${enable_udlimit}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
fi
#########
# See whether we should enable RTREE
AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
[Enable the RTREE extension]),
[enable_rtree=yes],[enable_rtree=no])
if test "${enable_rtree}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
fi
#########
# See whether we should enable the SESSION extension
AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
[Enable the SESSION extension]),
[enable_session=yes],[enable_session=no])
if test "${enable_session}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
fi
#########
# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
for option in $CFLAGS $CPPFLAGS
do
case $option in
+211 -353
View File
@@ -2,47 +2,24 @@
<head>
<title>The Lemon Parser Generator</title>
</head>
<body bgcolor='white'>
<h1 align='center'>The Lemon Parser Generator</h1>
<body bgcolor=white>
<h1 align=center>The Lemon Parser Generator</h1>
<p>Lemon is an LALR(1) parser generator for C.
It does the same job as "bison" and "yacc".
But Lemon is not a bison or yacc clone. Lemon
<p>Lemon is an LALR(1) parser generator for C or C++.
It does the same job as ``bison'' and ``yacc''.
But lemon is not another bison or yacc clone. It
uses a different grammar syntax which is designed to
reduce the number of coding errors. Lemon also uses a
parsing engine that is faster than yacc and
bison and which is both reentrant and threadsafe.
(Update: Since the previous sentence was written, bison
has also been updated so that it too can generate a
reentrant and threadsafe parser.)
Lemon also implements features that can be used
to eliminate resource leaks, making it suitable for use
reduce the number of coding errors. Lemon also uses a more
sophisticated parsing engine that is faster than yacc and
bison and which is both reentrant and thread-safe.
Furthermore, Lemon implements features that can be used
to eliminate resource leaks, making is suitable for use
in long-running programs such as graphical user interfaces
or embedded controllers.</p>
<p>This document is an introduction to the Lemon
parser generator.</p>
<h2>Security Note</h2>
<p>The language parser code created by Lemon is very robust and
is well-suited for use in internet-facing applications that need to
safely process maliciously crafted inputs.
<p>The "lemon.exe" command-line tool itself works great when given a valid
input grammar file and almost always gives helpful
error messages for malformed inputs. However, it is possible for
a malicious user to craft a grammar file that will cause
lemon.exe to crash.
We do not see this as a problem, as lemon.exe is not intended to be used
with hostile inputs.
To summarize:</p>
<ul>
<li>Parser code generated by lemon &rarr; Robust and secure
<li>The "lemon.exe" command line tool itself &rarr; Not so much
</ul>
<h2>Theory of Operation</h2>
<p>The main goal of Lemon is to translate a context free grammar (CFG)
@@ -58,8 +35,8 @@ Lemon comes with a default parser template which works fine for most
applications. But the user is free to substitute a different parser
template if desired.</p>
<p>Depending on command-line options, Lemon will generate up to
three output files.
<p>Depending on command-line options, Lemon will generate between
one and three files of outputs.
<ul>
<li>C code to implement the parser.
<li>A header file defining an integer ID for each terminal symbol.
@@ -67,18 +44,18 @@ three output files.
automaton.
</ul>
By default, all three of these output files are generated.
The header file is suppressed if the "-m" command-line option is
used and the report file is omitted when "-q" is selected.</p>
The header file is suppressed if the ``-m'' command-line option is
used and the report file is omitted when ``-q'' is selected.</p>
<p>The grammar specification file uses a ".y" suffix, by convention.
<p>The grammar specification file uses a ``.y'' suffix, by convention.
In the examples used in this document, we'll assume the name of the
grammar file is "gram.y". A typical use of Lemon would be the
grammar file is ``gram.y''. A typical use of Lemon would be the
following command:
<pre>
lemon gram.y
</pre>
This command will generate three output files named "gram.c",
"gram.h" and "gram.out".
This command will generate three output files named ``gram.c'',
``gram.h'' and ``gram.out''.
The first is C code to implement the parser. The second
is the header file that defines numerical values for all
terminal symbols, and the last is the report that explains
@@ -90,42 +67,43 @@ the states used by the parser automaton.</p>
You can obtain a list of the available command-line options together
with a brief explanation of what each does by typing
<pre>
lemon "-?"
lemon -?
</pre>
As of this writing, the following command-line options are supported:
<ul>
<li><b>-b</b>
Show only the basis for each parser state in the report file.
<li><b>-c</b>
Do not compress the generated action tables. The parser will be a
little larger and slower, but it will detect syntax errors sooner.
<li><b>-D<i>name</i></b>
Define C preprocessor macro <i>name</i>. This macro is usable by
"<tt><a href='#pifdef'>%ifdef</a></tt>" and
"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
in the grammar file.
<li><b>-g</b>
Do not generate a parser. Instead write the input grammar to standard
output with all comments, actions, and other extraneous text removed.
<li><b>-l</b>
Omit "#line" directives in the generated parser C code.
<li><b>-m</b>
Cause the output C source code to be compatible with the "makeheaders"
program.
<li><b>-p</b>
Display all conflicts that are resolved by
<a href='#precrules'>precedence rules</a>.
<li><b>-q</b>
Suppress generation of the report file.
<li><b>-r</b>
Do not sort or renumber the parser states as part of optimization.
<li><b>-s</b>
Show parser statistics before existing.
<li><b>-T<i>file</i></b>
Use <i>file</i> as the template for the generated C-code parser implementation.
<li><b>-x</b>
Print the Lemon version number.
<li><tt>-b</tt>
<li><tt>-c</tt>
<li><tt>-g</tt>
<li><tt>-m</tt>
<li><tt>-q</tt>
<li><tt>-s</tt>
<li><tt>-x</tt>
</ul>
The ``-b'' option reduces the amount of text in the report file by
printing only the basis of each parser state, rather than the full
configuration.
The ``-c'' option suppresses action table compression. Using -c
will make the parser a little larger and slower but it will detect
syntax errors sooner.
The ``-g'' option causes no output files to be generated at all.
Instead, the input grammar file is printed on standard output but
with all comments, actions and other extraneous text deleted. This
is a useful way to get a quick summary of a grammar.
The ``-m'' option causes the output C source file to be compatible
with the ``makeheaders'' program.
Makeheaders is a program that automatically generates header files
from C source code. When the ``-m'' option is used, the header
file is not output since the makeheaders program will take care
of generated all header files automatically.
The ``-q'' option suppresses the report file.
Using ``-s'' causes a brief summary of parser statistics to be
printed. Like this:
<pre>
Parser statistics: 74 terminals, 70 nonterminals, 179 rules
340 states, 2026 parser table entries, 0 conflicts
</pre>
Finally, the ``-x'' option causes Lemon to print its version number
and then stops without attempting to read the grammar or generate a parser.</p>
<h3>The Parser Interface</h3>
@@ -143,12 +121,12 @@ A new parser is created as follows:
</pre>
The ParseAlloc() routine allocates and initializes a new parser and
returns a pointer to it.
The actual data structure used to represent a parser is opaque &mdash;
The actual data structure used to represent a parser is opaque --
its internal structure is not visible or usable by the calling routine.
For this reason, the ParseAlloc() routine returns a pointer to void
rather than a pointer to some particular structure.
The sole argument to the ParseAlloc() routine is a pointer to the
subroutine used to allocate memory. Typically this means malloc().</p>
subroutine used to allocate memory. Typically this means ``malloc()''.</p>
<p>After a program is finished using a parser, it can reclaim all
memory allocated by that parser by calling
@@ -168,24 +146,23 @@ once for each token:
</pre>
The first argument to the Parse() routine is the pointer returned by
ParseAlloc().
The second argument is a small positive integer that tells the parser the
The second argument is a small positive integer that tells the parse the
type of the next token in the data stream.
There is one token type for each terminal symbol in the grammar.
The gram.h file generated by Lemon contains #define statements that
map symbolic terminal symbol names into appropriate integer values.
A value of 0 for the second argument is a special flag to the
parser to indicate that the end of input has been reached.
(A value of 0 for the second argument is a special flag to the
parser to indicate that the end of input has been reached.)
The third argument is the value of the given token. By default,
the type of the third argument is "void*", but the grammar will
the type of the third argument is integer, but the grammar will
usually redefine this type to be some kind of structure.
Typically the second argument will be a broad category of tokens
such as "identifier" or "number" and the third argument will
such as ``identifier'' or ``number'' and the third argument will
be the name of the identifier or the value of the number.</p>
<p>The Parse() function may have either three or four arguments,
depending on the grammar. If the grammar specification file requests
it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
the Parse() function will have a fourth parameter that can be
depending on the grammar. If the grammar specification file request
it, the Parse() function will have a fourth parameter that can be
of any type chosen by the programmer. The parser doesn't do anything
with this argument except to pass it through to action routines.
This is a convenient mechanism for passing state information down
@@ -194,20 +171,20 @@ to the action routines without having to use global variables.</p>
<p>A typical use of a Lemon parser might look something like the
following:
<pre>
1 ParseTree *ParseFile(const char *zFilename){
2 Tokenizer *pTokenizer;
3 void *pParser;
4 Token sToken;
5 int hTokenId;
6 ParserState sState;
7
8 pTokenizer = TokenizerCreate(zFilename);
9 pParser = ParseAlloc( malloc );
10 InitParserState(&amp;sState);
11 while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
12 Parse(pParser, hTokenId, sToken, &amp;sState);
01 ParseTree *ParseFile(const char *zFilename){
02 Tokenizer *pTokenizer;
03 void *pParser;
04 Token sToken;
05 int hTokenId;
06 ParserState sState;
07
08 pTokenizer = TokenizerCreate(zFilename);
09 pParser = ParseAlloc( malloc );
10 InitParserState(&sState);
11 while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
12 Parse(pParser, hTokenId, sToken, &sState);
13 }
14 Parse(pParser, 0, sToken, &amp;sState);
14 Parse(pParser, 0, sToken, &sState);
15 ParseFree(pParser, free );
16 TokenizerFree(pTokenizer);
17 return sState.treeRoot;
@@ -215,15 +192,15 @@ following:
</pre>
This example shows a user-written routine that parses a file of
text and returns a pointer to the parse tree.
(All error-handling code is omitted from this example to keep it
(We've omitted all error-handling from this example to keep it
simple.)
We assume the existence of some kind of tokenizer which is created
using TokenizerCreate() on line 8 and deleted by TokenizerFree()
on line 16. The GetNextToken() function on line 11 retrieves the
next token from the input file and puts its type in the
next token from the input file and puts its type in the
integer variable hTokenId. The sToken variable is assumed to be
some kind of structure that contains details about each token,
such as its complete text, what line it occurs on, etc.</p>
such as its complete text, what line it occurs on, etc. </p>
<p>This example also assumes the existence of structure of type
ParserState that holds state information about a particular parse.
@@ -240,7 +217,7 @@ tree.</p>
<pre>
ParseFile(){
pParser = ParseAlloc( malloc );
while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
Parse(pParser, hTokenId, sToken);
}
Parse(pParser, 0, sToken);
@@ -286,12 +263,6 @@ with prior yacc and bison experience.
But after years of experience using Lemon, I firmly
believe that the Lemon way of doing things is better.</p>
<p><i>Updated as of 2016-02-16:</i>
The text above was written in the 1990s.
We are told that Bison has lately been enhanced to support the
tokenizer-calls-parser paradigm used by Lemon, and to obviate the
need for global variables.</p>
<h2>Input File Syntax</h2>
<p>The main purpose of the grammar specification file for Lemon is
@@ -300,25 +271,25 @@ specifies additional information Lemon requires to do its job.
Most of the work in using Lemon is in writing an appropriate
grammar file.</p>
<p>The grammar file for Lemon is, for the most part, free format.
<p>The grammar file for lemon is, for the most part, free format.
It does not have sections or divisions like yacc or bison. Any
declaration can occur at any point in the file.
Lemon ignores whitespace (except where it is needed to separate
tokens), and it honors the same commenting conventions as C and C++.</p>
tokens) and it honors the same commenting conventions as C and C++.</p>
<h3>Terminals and Nonterminals</h3>
<p>A terminal symbol (token) is any string of alphanumeric
and/or underscore characters
that begins with an uppercase letter.
and underscore characters
that begins with an upper case letter.
A terminal can contain lowercase letters after the first character,
but the usual convention is to make terminals all uppercase.
but the usual convention is to make terminals all upper case.
A nonterminal, on the other hand, is any string of alphanumeric
and underscore characters than begins with a lowercase letter.
Again, the usual convention is to make nonterminals use all lowercase
letters.</p>
and underscore characters than begins with a lower case letter.
Again, the usual convention is to make nonterminals use all lower
case letters.</p>
<p>In Lemon, terminal and nonterminal symbols do not need to
<p>In Lemon, terminal and nonterminal symbols do not need to
be declared or identified in a separate section of the grammar file.
Lemon is able to generate a list of all terminals and nonterminals
by examining the grammar rules, and it can always distinguish a
@@ -336,14 +307,13 @@ must have alphanumeric names.</p>
<p>The main component of a Lemon grammar file is a sequence of grammar
rules.
Each grammar rule consists of a nonterminal symbol followed by
the special symbol "::=" and then a list of terminals and/or nonterminals.
the special symbol ``::='' and then a list of terminals and/or nonterminals.
The rule is terminated by a period.
The list of terminals and nonterminals on the right-hand side of the
rule can be empty.
Rules can occur in any order, except that the left-hand side of the
first rule is assumed to be the start symbol for the grammar (unless
specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
directive described below.)
specified otherwise using the <tt>%start</tt> directive described below.)
A typical sequence of grammar rules might look something like this:
<pre>
expr ::= expr PLUS expr.
@@ -353,9 +323,9 @@ A typical sequence of grammar rules might look something like this:
</pre>
</p>
<p>There is one non-terminal in this example, "expr", and five
terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
"RPAREN" and "VALUE".</p>
<p>There is one non-terminal in this example, ``expr'', and five
terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
``RPAREN'' and ``VALUE''.</p>
<p>Like yacc and bison, Lemon allows the grammar to specify a block
of C code that will be executed whenever a grammar rule is reduced
@@ -371,22 +341,22 @@ For example:
<p>In order to be useful, grammar actions must normally be linked to
their associated grammar rules.
In yacc and bison, this is accomplished by embedding a "$$" in the
In yacc and bison, this is accomplished by embedding a ``$$'' in the
action to stand for the value of the left-hand side of the rule and
symbols "$1", "$2", and so forth to stand for the value of
symbols ``$1'', ``$2'', and so forth to stand for the value of
the terminal or nonterminal at position 1, 2 and so forth on the
right-hand side of the rule.
This idea is very powerful, but it is also very error-prone. The
single most common source of errors in a yacc or bison grammar is
to miscount the number of symbols on the right-hand side of a grammar
rule and say "$7" when you really mean "$8".</p>
rule and say ``$7'' when you really mean ``$8''.</p>
<p>Lemon avoids the need to count grammar symbols by assigning symbolic
names to each symbol in a grammar rule and then using those symbolic
names in the action.
In yacc or bison, one would write this:
<pre>
expr -&gt; expr PLUS expr { $$ = $1 + $3; };
expr -> expr PLUS expr { $$ = $1 + $3; };
</pre>
But in Lemon, the same rule becomes the following:
<pre>
@@ -409,7 +379,7 @@ For example, the rule
<pre>
expr(A) ::= expr(B) PLUS expr(C). { A = B; }
</pre>
will generate an error because the linking symbol "C" is used
will generate an error because the linking symbol ``C'' is used
in the grammar rule but not in the reduce action.</p>
<p>The Lemon notation for linking grammar rules to reduce actions
@@ -417,7 +387,6 @@ also facilitates the use of destructors for reclaiming memory
allocated by the values of terminals and nonterminals on the
right-hand side of a rule.</p>
<a name='precrules'></a>
<h3>Precedence Rules</h3>
<p>Lemon resolves parsing ambiguities in exactly the same way as
@@ -426,14 +395,11 @@ of the shift, and a reduce-reduce conflict is resolved by reducing
whichever rule comes first in the grammar file.</p>
<p>Just like in
yacc and bison, Lemon allows a measure of control
over the resolution of parsing conflicts using precedence rules.
yacc and bison, Lemon allows a measure of control
over the resolution of paring conflicts using precedence rules.
A precedence value can be assigned to any terminal symbol
using the
<tt><a href='#pleft'>%left</a></tt>,
<tt><a href='#pright'>%right</a></tt> or
<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives. Terminal symbols
mentioned in earlier directives have a lower precedence than
using the %left, %right or %nonassoc directives. Terminal symbols
mentioned in earlier directives have a lower precedence that
terminal symbols mentioned in later directives. For example:</p>
<p><pre>
@@ -509,29 +475,29 @@ as follows:
<li> If the precedence of the token to be shifted is greater than
the precedence of the rule to reduce, then resolve in favor
of the shift. No parsing conflict is reported.
<li> If the precedence of the token to be shifted is less than the
<li> If the precedence of the token it be shifted is less than the
precedence of the rule to reduce, then resolve in favor of the
reduce action. No parsing conflict is reported.
<li> If the precedences are the same and the shift token is
right-associative, then resolve in favor of the shift.
No parsing conflict is reported.
<li> If the precedences are the same and the shift token is
<li> If the precedences are the same the shift token is
left-associative, then resolve in favor of the reduce.
No parsing conflict is reported.
<li> Otherwise, resolve the conflict by doing the shift, and
report a parsing conflict.
<li> Otherwise, resolve the conflict by doing the shift and
report the parsing conflict.
</ul>
Reduce-reduce conflicts are resolved this way:
<ul>
<li> If either reduce rule
<li> If either reduce rule
lacks precedence information, then resolve in favor of the
rule that appears first in the grammar, and report a parsing
rule that appears first in the grammar and report a parsing
conflict.
<li> If both rules have precedence and the precedence is different,
<li> If both rules have precedence and the precedence is different
then resolve the dispute in favor of the rule with the highest
precedence, and do not report a conflict.
precedence and do not report a conflict.
<li> Otherwise, resolve the conflict by reducing by the rule that
appears first in the grammar, and report a parsing conflict.
appears first in the grammar and report a parsing conflict.
</ul>
<h3>Special Directives</h3>
@@ -540,84 +506,73 @@ Reduce-reduce conflicts are resolved this way:
directives. We've described all the grammar rules, so now we'll
talk about the special directives.</p>
<p>Directives in Lemon can occur in any order. You can put them before
the grammar rules, or after the grammar rules, or in the midst of the
<p>Directives in lemon can occur in any order. You can put them before
the grammar rules, or after the grammar rules, or in the mist of the
grammar rules. It doesn't matter. The relative order of
directives used to assign precedence to terminals is important, but
other than that, the order of directives in Lemon is arbitrary.</p>
<p>Lemon supports the following special directives:
<ul>
<li><tt><a href='#pcode'>%code</a></tt>
<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
<li><tt><a href='#default_type'>%default_type</a></tt>
<li><tt><a href='#destructor'>%destructor</a></tt>
<li><tt><a href='#pifdef'>%endif</a></tt>
<li><tt><a href='#extraarg'>%extra_argument</a></tt>
<li><tt><a href='#pfallback'>%fallback</a></tt>
<li><tt><a href='#pifdef'>%ifdef</a></tt>
<li><tt><a href='#pifdef'>%ifndef</a></tt>
<li><tt><a href='#pinclude'>%include</a></tt>
<li><tt><a href='#pleft'>%left</a></tt>
<li><tt><a href='#pname'>%name</a></tt>
<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
<li><tt><a href='#pright'>%right</a></tt>
<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
<li><tt><a href='#stack_size'>%stack_size</a></tt>
<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
<li><tt><a href='#token_class'>%token_class</a></tt>
<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
<li><tt><a href='#token_type'>%token_type</a></tt>
<li><tt><a href='#ptype'>%type</a></tt>
<li><tt><a href='#pwildcard'>%wildcard</a></tt>
<li><tt>%code</tt>
<li><tt>%default_destructor</tt>
<li><tt>%default_type</tt>
<li><tt>%destructor</tt>
<li><tt>%extra_argument</tt>
<li><tt>%include</tt>
<li><tt>%left</tt>
<li><tt>%name</tt>
<li><tt>%nonassoc</tt>
<li><tt>%parse_accept</tt>
<li><tt>%parse_failure </tt>
<li><tt>%right</tt>
<li><tt>%stack_overflow</tt>
<li><tt>%stack_size</tt>
<li><tt>%start_symbol</tt>
<li><tt>%syntax_error</tt>
<li><tt>%token_destructor</tt>
<li><tt>%token_prefix</tt>
<li><tt>%token_type</tt>
<li><tt>%type</tt>
</ul>
Each of these directives will be described separately in the
following sections:</p>
<a name='pcode'></a>
<h4>The <tt>%code</tt> directive</h4>
<p>The <tt>%code</tt> directive is used to specify additional C code that
<p>The %code directive is used to specify addition C/C++ code that
is added to the end of the main output file. This is similar to
the <tt><a href='#pinclude'>%include</a></tt> directive except that
<tt>%include</tt> is inserted at the beginning of the main output file.</p>
the %include directive except that %include is inserted at the
beginning of the main output file.</p>
<p><tt>%code</tt> is typically used to include some action routines or perhaps
a tokenizer or even the "main()" function
as part of the output file.</p>
<p>%code is typically used to include some action routines or perhaps
a tokenizer as part of the output file.</p>
<a name='default_destructor'></a>
<h4>The <tt>%default_destructor</tt> directive</h4>
<p>The <tt>%default_destructor</tt> directive specifies a destructor to
<p>The %default_destructor directive specifies a destructor to
use for non-terminals that do not have their own destructor
specified by a separate <tt>%destructor</tt> directive. See the documentation
on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
additional information.</p>
specified by a separate %destructor directive. See the documentation
on the %destructor directive below for additional information.</p>
<p>In some grammars, many different non-terminal symbols have the
same data type and hence the same destructor. This directive is
a convenient way to specify the same destructor for all those
<p>In some grammers, many different non-terminal symbols have the
same datatype and hence the same destructor. This directive is
a convenience way to specify the same destructor for all those
non-terminals using a single statement.</p>
<a name='default_type'></a>
<h4>The <tt>%default_type</tt> directive</h4>
<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
symbols that do not have their own data type defined using a separate
<tt><a href='#ptype'>%type</a></tt> directive.</p>
<p>The %default_type directive specifies the datatype of non-terminal
symbols that do no have their own datatype defined using a separate
%type directive. See the documentation on %type below for addition
information.</p>
<a name='destructor'></a>
<h4>The <tt>%destructor</tt> directive</h4>
<p>The <tt>%destructor</tt> directive is used to specify a destructor for
<p>The %destructor directive is used to specify a destructor for
a non-terminal symbol.
(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
directive which is used to specify a destructor for terminal symbols.)</p>
(See also the %token_destructor directive which is used to
specify a destructor for terminal symbols.)</p>
<p>A non-terminal's destructor is called to dispose of the
non-terminal's value whenever the non-terminal is popped from
@@ -638,33 +593,33 @@ or other resources held by that non-terminal.</p>
%destructor nt { free($$); }
nt(A) ::= ID NUM. { A = malloc( 100 ); }
</pre>
This example is a bit contrived, but it serves to illustrate how
This example is a bit contrived but it serves to illustrate how
destructors work. The example shows a non-terminal named
"nt" that holds values of type "void*". When the rule for
an "nt" reduces, it sets the value of the non-terminal to
``nt'' that holds values of type ``void*''. When the rule for
an ``nt'' reduces, it sets the value of the non-terminal to
space obtained from malloc(). Later, when the nt non-terminal
is popped from the stack, the destructor will fire and call
free() on this malloced space, thus avoiding a memory leak.
(Note that the symbol "$$" in the destructor code is replaced
(Note that the symbol ``$$'' in the destructor code is replaced
by the value of the non-terminal.)</p>
<p>It is important to note that the value of a non-terminal is passed
to the destructor whenever the non-terminal is removed from the
stack, unless the non-terminal is used in a C-code action. If
the non-terminal is used by C-code, then it is assumed that the
C-code will take care of destroying it.
More commonly, the value is used to build some
larger structure, and we don't want to destroy it, which is why
C-code will take care of destroying it if it should really
be destroyed. More commonly, the value is used to build some
larger structure and we don't want to destroy it, which is why
the destructor is not called in this circumstance.</p>
<p>Destructors help avoid memory leaks by automatically freeing
allocated objects when they go out of scope.
<p>By appropriate use of destructors, it is possible to
build a parser using Lemon that can be used within a long-running
program, such as a GUI, that will not leak memory or other resources.
To do the same using yacc or bison is much more difficult.</p>
<a name='extraarg'></a>
<h4>The <tt>%extra_argument</tt> directive</h4>
The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
The %extra_argument directive instructs Lemon to add a 4th parameter
to the parameter list of the Parse() function it generates. Lemon
doesn't do anything itself with this extra argument, but it does
make the argument available to C-code action routines, destructors,
@@ -675,71 +630,19 @@ and so forth. For example, if the grammar file contains:</p>
</pre></p>
<p>Then the Parse() function generated will have an 4th parameter
of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of the 4th parameter
of type ``MyStruct*'' and all action routines will have access to
a variable named ``pAbc'' that is the value of the 4th parameter
in the most recent call to Parse().</p>
<a name='pfallback'></a>
<h4>The <tt>%fallback</tt> directive</h4>
<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
or more tokens. The alternative meaning is tried if the original token
would have generated a syntax error.</p>
<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
The SQL language contains a large assortment of keywords, each of which
appears as a different token to the language parser. SQL contains so
many keywords that it can be difficult for programmers to keep up with
them all. Programmers will, therefore, sometimes mistakenly use an
obscure language keyword for an identifier. The <tt>%fallback</tt> directive
provides a mechanism to tell the parser: "If you are unable to parse
this keyword, try treating it as an identifier instead."</p>
<p>The syntax of <tt>%fallback</tt> is as follows:
<blockquote>
<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
</blockquote></p>
<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
names terminated by a period.
The first token name is the fallback token &mdash; the
token to which all the other tokens fall back to. The second and subsequent
arguments are tokens which fall back to the token identified by the first
argument.</p>
<a name='pifdef'></a>
<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>
<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
just not as general.
Each of these directives must begin at the left margin. No whitespace
is allowed between the "%" and the directive name.</p>
<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
"<tt>%endif</tt>" is
ignored unless the "-DMACRO" command-line option is used. Grammar text
betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
included except when the "-DMACRO" command-line option is used.</p>
<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
be a single preprocessor symbol name, not a general expression.
There is no "<tt>%else</tt>" directive.</p>
<a name='pinclude'></a>
<h4>The <tt>%include</tt> directive</h4>
<p>The <tt>%include</tt> directive specifies C code that is included at the
top of the generated parser. You can include any text you want &mdash;
<p>The %include directive specifies C code that is included at the
top of the generated parser. You can include any text you want --
the Lemon parser generator copies it blindly. If you have multiple
<tt>%include</tt> directives in your grammar file, their values are concatenated
so that all <tt>%include</tt> code ultimately appears near the top of the
generated parser, in the same order as it appeared in the grammar.</p>
%include directives in your grammar file the value of the last
%include directive overwrites all the others.</p.
<p>The <tt>%include</tt> directive is very handy for getting some extra #include
<p>The %include directive is very handy for getting some extra #include
preprocessor statements at the beginning of the generated parser.
For example:</p>
@@ -748,19 +651,16 @@ For example:</p>
</pre></p>
<p>This might be needed, for example, if some of the C actions in the
grammar call functions that are prototyped in unistd.h.</p>
grammar call functions that are prototyed in unistd.h.</p>
<a name='pleft'></a>
<h4>The <tt>%left</tt> directive</h4>
The <tt>%left</tt> directive is used (along with the
<tt><a href='#pright'>%right</a></tt> and
<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
precedences of terminal symbols.
Every terminal symbol whose name appears after
a <tt>%left</tt> directive but before the next period (".") is
The %left directive is used (along with the %right and
%nonassoc directives) to declare precedences of terminal
symbols. Every terminal symbol whose name appears after
a %left directive but before the next period (``.'') is
given the same left-associative precedence value. Subsequent
<tt>%left</tt> directives have higher precedence. For example:</p>
%left directives have higher precedence. For example:</p>
<p><pre>
%left AND.
@@ -771,21 +671,19 @@ given the same left-associative precedence value. Subsequent
%right EXP NOT.
</pre></p>
<p>Note the period that terminates each <tt>%left</tt>,
<tt>%right</tt> or <tt>%nonassoc</tt>
<p>Note the period that terminates each %left, %right or %nonassoc
directive.</p>
<p>LALR(1) grammars can get into a situation where they require
a large amount of stack space if you make heavy use or right-associative
operators. For this reason, it is recommended that you use <tt>%left</tt>
rather than <tt>%right</tt> whenever possible.</p>
operators. For this reason, it is recommended that you use %left
rather than %right whenever possible.</p>
<a name='pname'></a>
<h4>The <tt>%name</tt> directive</h4>
<p>By default, the functions generated by Lemon all begin with the
five-character string "Parse". You can change this string to something
different using the <tt>%name</tt> directive. For instance:</p>
five-character string ``Parse''. You can change this string to something
different using the %name directive. For instance:</p>
<p><pre>
%name Abcde
@@ -799,23 +697,20 @@ functions named
<li> AbcdeTrace(), and
<li> Abcde().
</ul>
The <tt>%name</tt> directive allows you to generate two or more different
parsers and link them all into the same executable.</p>
The %name directive allows you to generator two or more different
parsers and link them all into the same executable.
</p>
<a name='pnonassoc'></a>
<h4>The <tt>%nonassoc</tt> directive</h4>
<p>This directive is used to assign non-associative precedence to
one or more terminal symbols. See the section on
<a href='#precrules'>precedence rules</a>
or on the <tt><a href='#pleft'>%left</a></tt> directive
for additional information.</p>
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
<a name='parse_accept'></a>
<h4>The <tt>%parse_accept</tt> directive</h4>
<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
executed whenever the parser accepts its input string. To "accept"
<p>The %parse_accept directive specifies a block of C code that is
executed whenever the parser accepts its input string. To ``accept''
an input string means that the parser was able to process all tokens
without error.</p>
@@ -827,10 +722,10 @@ without error.</p>
}
</pre></p>
<a name='parse_failure'></a>
<h4>The <tt>%parse_failure</tt> directive</h4>
<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
<p>The %parse_failure directive specifies a block of C code that
is executed whenever the parser fails complete. This code is not
executed until the parser has tried and failed to resolve an input
error using is usual error recovery strategy. The routine is
@@ -842,18 +737,15 @@ only invoked when parsing is unable to continue.</p>
}
</pre></p>
<a name='pright'></a>
<h4>The <tt>%right</tt> directive</h4>
<p>This directive is used to assign right-associative precedence to
one or more terminal symbols. See the section on
<a href='#precrules'>precedence rules</a>
or on the <a href='#pleft'>%left</a> directive for additional information.</p>
one or more terminal symbols. See the section on precedence rules
or on the %left directive for additional information.</p>
<a name='stack_overflow'></a>
<h4>The <tt>%stack_overflow</tt> directive</h4>
<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
<p>The %stack_overflow directive specifies a block of C code that
is executed if the parser's internal stack ever overflows. Typically
this just prints an error message. After a stack overflow, the parser
will be unable to continue and must be reset.</p>
@@ -866,7 +758,7 @@ will be unable to continue and must be reset.</p>
<p>You can help prevent parser stack overflows by avoiding the use
of right recursion and right-precedence operators in your grammar.
Use left recursion and and left-precedence operators instead to
Use left recursion and and left-precedence operators instead, to
encourage rules to reduce sooner and keep the stack size down.
For example, do rules like this:
<pre>
@@ -877,69 +769,49 @@ Not like this:
<pre>
list ::= element list. // right-recursion. Bad!
list ::= .
</pre></p>
</pre>
<a name='stack_size'></a>
<h4>The <tt>%stack_size</tt> directive</h4>
<p>If stack overflow is a problem and you can't resolve the trouble
by using left-recursion, then you might want to increase the size
of the parser's stack using this directive. Put an positive integer
after the <tt>%stack_size</tt> directive and Lemon will generate a parse
after the %stack_size directive and Lemon will generate a parse
with a stack of the requested size. The default value is 100.</p>
<p><pre>
%stack_size 2000
</pre></p>
<a name='start_symbol'></a>
<h4>The <tt>%start_symbol</tt> directive</h4>
<p>By default, the start symbol for the grammar that Lemon generates
<p>By default, the start-symbol for the grammar that Lemon generates
is the first non-terminal that appears in the grammar file. But you
can choose a different start symbol using the
<tt>%start_symbol</tt> directive.</p>
can choose a different start-symbol using the %start_symbol directive.</p>
<p><pre>
%start_symbol prog
</pre></p>
<a name='syntax_error'></a>
<h4>The <tt>%syntax_error</tt> directive</h4>
<p>See <a href='#error_processing'>Error Processing</a>.</p>
<a name='token_class'></a>
<h4>The <tt>%token_class</tt> directive</h4>
<p>Undocumented. Appears to be related to the MULTITERMINAL concept.
<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
<a name='token_destructor'></a>
<h4>The <tt>%token_destructor</tt> directive</h4>
<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
symbol. (See the description of the
<tt><a href='%destructor'>%destructor</a></tt> directive above.)
The <tt>%token_destructor</tt> directive does the same thing
for all terminal symbols.</p>
<p>The %destructor directive assigns a destructor to a non-terminal
symbol. (See the description of the %destructor directive above.)
This directive does the same thing for all terminal symbols.</p>
<p>Unlike non-terminal symbols which may each have a different data type
for their values, terminals all use the same data type (defined by
the <tt><a href='#token_type'>%token_type</a></tt> directive)
and so they use a common destructor.
Other than that, the token destructor works just like the non-terminal
the %token_type directive) and so they use a common destructor. Other
than that, the token destructor works just like the non-terminal
destructors.</p>
<a name='token_prefix'></a>
<h4>The <tt>%token_prefix</tt> directive</h4>
<p>Lemon generates #defines that assign small integer constants
to each terminal symbol in the grammar. If desired, Lemon will
add a prefix specified by this directive
to each of the #defines it generates.</p>
<p>So if the default output of Lemon looked like this:
to each of the #defines it generates.
So if the default output of Lemon looked like this:
<pre>
#define AND 1
#define MINUS 2
@@ -956,9 +828,8 @@ to cause Lemon to produce these symbols instead:
#define TOKEN_MINUS 2
#define TOKEN_OR 3
#define TOKEN_PLUS 4
</pre></p>
</pre>
<a name='token_type'></a><a name='ptype'></a>
<h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
<p>These directives are used to specify the data types for values
@@ -974,10 +845,10 @@ token structure. Like this:</p>
</pre></p>
<p>If the data type of terminals is not specified, the default value
is "void*".</p>
is ``int''.</p>
<p>Non-terminal symbols can each have their own data types. Typically
the data type of a non-terminal is a pointer to the root of a parse tree
the data type of a non-terminal is a pointer to the root of a parse-tree
structure that contains all information about that non-terminal.
For example:</p>
@@ -995,18 +866,6 @@ non-terminal whose data type requires 1K of storage, then your 100
entry parser stack will require 100K of heap space. If you are willing
and able to pay that price, fine. You just need to know.</p>
<a name='pwildcard'></a>
<h4>The <tt>%wildcard</tt> directive</h4>
<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
period. This directive specifies that the identified token should
match any input token.</p>
<p>When the generated parser has the choice of matching an input against
the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no alternatives.</p>
<a name='error_processing'></a>
<h3>Error Processing</h3>
<p>After extensive experimentation over several years, it has been
@@ -1014,21 +873,20 @@ discovered that the error recovery strategy used by yacc is about
as good as it gets. And so that is what Lemon uses.</p>
<p>When a Lemon-generated parser encounters a syntax error, it
first invokes the code specified by the <tt>%syntax_error</tt> directive, if
first invokes the code specified by the %syntax_error directive, if
any. It then enters its error recovery strategy. The error recovery
strategy is to begin popping the parsers stack until it enters a
state where it is permitted to shift a special non-terminal symbol
named "error". It then shifts this non-terminal and continues
parsing. The <tt>%syntax_error</tt> routine will not be called again
named ``error''. It then shifts this non-terminal and continues
parsing. But the %syntax_error routine will not be called again
until at least three new tokens have been successfully shifted.</p>
<p>If the parser pops its stack until the stack is empty, and it still
is unable to shift the error symbol, then the
<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
is unable to shift the error symbol, then the %parse_failed routine
is invoked and the parser resets itself to its start state, ready
to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
"error" non-terminal in your grammar.</p>
first syntax error, of course, if there are no instances of the
``error'' non-terminal in your grammar.</p>
</body>
</html>
-8
View File
@@ -1,8 +0,0 @@
## Loadable Extensions
Various [loadable extensions](https://www.sqlite.org/loadext.html) for
SQLite are found in subfolders.
Most subfolders are dedicated to a single loadable extension (for
example FTS5, or RTREE). But the misc/ subfolder contains a collection
of smaller single-file extensions.
+2
View File
@@ -0,0 +1,2 @@
Version loadable extensions to SQLite are found in subfolders
of this folder.
+5 -11
View File
@@ -1636,7 +1636,6 @@ void sqlite3async_run(void){
** Control/configure the asynchronous IO system.
*/
int sqlite3async_control(int op, ...){
int rc = SQLITE_OK;
va_list ap;
va_start(ap, op);
switch( op ){
@@ -1646,8 +1645,7 @@ int sqlite3async_control(int op, ...){
&& eWhen!=SQLITEASYNC_HALT_NOW
&& eWhen!=SQLITEASYNC_HALT_IDLE
){
rc = SQLITE_MISUSE;
break;
return SQLITE_MISUSE;
}
async.eHalt = eWhen;
async_mutex_enter(ASYNC_MUTEX_QUEUE);
@@ -1659,8 +1657,7 @@ int sqlite3async_control(int op, ...){
case SQLITEASYNC_DELAY: {
int iDelay = va_arg(ap, int);
if( iDelay<0 ){
rc = SQLITE_MISUSE;
break;
return SQLITE_MISUSE;
}
async.ioDelay = iDelay;
break;
@@ -1671,8 +1668,7 @@ int sqlite3async_control(int op, ...){
async_mutex_enter(ASYNC_MUTEX_QUEUE);
if( async.nFile || async.pQueueFirst ){
async_mutex_leave(ASYNC_MUTEX_QUEUE);
rc = SQLITE_MISUSE;
break;
return SQLITE_MISUSE;
}
async.bLockFiles = bLock;
async_mutex_leave(ASYNC_MUTEX_QUEUE);
@@ -1696,11 +1692,9 @@ int sqlite3async_control(int op, ...){
}
default:
rc = SQLITE_ERROR;
break;
return SQLITE_ERROR;
}
va_end(ap);
return rc;
return SQLITE_OK;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ASYNCIO) */
-83
View File
@@ -1,83 +0,0 @@
## SQLite Expert Extension
This folder contains code for a simple system to propose useful indexes
given a database and a set of SQL queries. It works as follows:
1. The user database schema is copied to a temporary database.
1. All SQL queries are prepared against the temporary database.
Information regarding the WHERE and ORDER BY clauses, and other query
features that affect index selection are recorded.
1. The information gathered in step 2 is used to create candidate
indexes - indexes that the planner might have made use of in the previous
step, had they been available.
1. A subset of the data in the user database is used to generate statistics
for all existing indexes and the candidate indexes generated in step 3
above.
1. The SQL queries are prepared a second time. If the planner uses any
of the indexes created in step 3, they are recommended to the user.
# C API
The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
as follows:
1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
A database handle opened by the user is passed as an argument.
1. The sqlite3expert object is configured with one or more SQL statements
by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
specify a single SQL statement, or multiple statements separated by
semi-colons.
1. Optionally, the **sqlite3\_expert\_config()** API may be used to
configure the size of the data subset used to generate index statistics.
Using a smaller subset of the data can speed up the analysis.
1. **sqlite3\_expert\_analyze()** is called to run the analysis.
1. One or more calls are made to **sqlite3\_expert\_report()** to extract
components of the results of the analysis.
1. **sqlite3\_expert\_destroy()** is called to free all resources.
Refer to comments in sqlite3expert.h for further details.
# sqlite3_expert application
The file "expert.c" contains the code for a command line application that
uses the API described above. It can be compiled with (for example):
<pre>
gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
</pre>
Assuming the database is named "test.db", it can then be run to analyze a
single query:
<pre>
./sqlite3_expert -sql &lt;sql-query&gt; test.db
</pre>
Or an entire text file worth of queries with:
<pre>
./sqlite3_expert -file &lt;text-file&gt; test.db
</pre>
By default, sqlite3\_expert generates index statistics using all the data in
the user database. For a large database, this may be prohibitively time
consuming. The "-sample" option may be used to configure sqlite3\_expert to
generate statistics based on an integer percentage of the user database as
follows:
<pre>
# Generate statistics based on 25% of the user database rows:
./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
# Do not generate any statistics at all:
./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
</pre>
-155
View File
@@ -1,155 +0,0 @@
/*
** 2017 April 07
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#include <sqlite3.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "sqlite3expert.h"
static void option_requires_argument(const char *zOpt){
fprintf(stderr, "Option requires an argument: %s\n", zOpt);
exit(-3);
}
static int option_integer_arg(const char *zVal){
return atoi(zVal);
}
static void usage(char **argv){
fprintf(stderr, "\n");
fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "Options are:\n");
fprintf(stderr, " -sql SQL (analyze SQL statements passed as argument)\n");
fprintf(stderr, " -file FILE (read SQL statements from file FILE)\n");
fprintf(stderr, " -verbose LEVEL (integer verbosity level. default 1)\n");
fprintf(stderr, " -sample PERCENT (percent of db to sample. default 100)\n");
exit(-1);
}
static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
FILE *in = fopen(zFile, "rb");
long nIn;
size_t nRead;
char *pBuf;
int rc;
if( in==0 ){
*pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
return SQLITE_ERROR;
}
fseek(in, 0, SEEK_END);
nIn = ftell(in);
rewind(in);
pBuf = sqlite3_malloc64( nIn+1 );
nRead = fread(pBuf, nIn, 1, in);
fclose(in);
if( nRead!=1 ){
sqlite3_free(pBuf);
*pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
return SQLITE_ERROR;
}
pBuf[nIn] = 0;
rc = sqlite3_expert_sql(p, pBuf, pzErr);
sqlite3_free(pBuf);
return rc;
}
int main(int argc, char **argv){
const char *zDb;
int rc = 0;
char *zErr = 0;
int i;
int iVerbose = 1; /* -verbose option */
sqlite3 *db = 0;
sqlite3expert *p = 0;
if( argc<2 ) usage(argv);
zDb = argv[argc-1];
if( zDb[0]=='-' ) usage(argv);
rc = sqlite3_open(zDb, &db);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
exit(-2);
}
p = sqlite3_expert_new(db, &zErr);
if( p==0 ){
fprintf(stderr, "Cannot run analysis: %s\n", zErr);
rc = 1;
}else{
for(i=1; i<(argc-1); i++){
char *zArg = argv[i];
if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
int nArg = (int)strlen(zArg);
if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
if( ++i==(argc-1) ) option_requires_argument("-file");
rc = readSqlFromFile(p, argv[i], &zErr);
}
else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
if( ++i==(argc-1) ) option_requires_argument("-sql");
rc = sqlite3_expert_sql(p, argv[i], &zErr);
}
else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
int iSample;
if( ++i==(argc-1) ) option_requires_argument("-sample");
iSample = option_integer_arg(argv[i]);
sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
}
else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
if( ++i==(argc-1) ) option_requires_argument("-verbose");
iVerbose = option_integer_arg(argv[i]);
}
else{
usage(argv);
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3_expert_analyze(p, &zErr);
}
if( rc==SQLITE_OK ){
int nQuery = sqlite3_expert_count(p);
if( iVerbose>0 ){
const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
fprintf(stdout, "-- Candidates -------------------------------\n");
fprintf(stdout, "%s\n", zCand);
}
for(i=0; i<nQuery; i++){
const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
if( zIdx==0 ) zIdx = "(no new indexes)\n";
if( iVerbose>0 ){
fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
fprintf(stdout, "%s\n\n", zSql);
}
fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
}
}else{
fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
}
sqlite3_expert_destroy(p);
sqlite3_free(zErr);
return rc;
}
-382
View File
@@ -1,382 +0,0 @@
# 2009 Nov 11
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the CLI shell tool. Specifically,
# the ".recommend" command.
#
#
# Test plan:
#
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set testprefix expert1
if {[info commands sqlite3_expert_new]==""} {
finish_test
return
}
set CLI [test_binary_name sqlite3]
set CMD [test_binary_name sqlite3_expert]
proc squish {txt} {
regsub -all {[[:space:]]+} $txt { }
}
proc do_setup_rec_test {tn setup sql res} {
reset_db
db eval $setup
uplevel [list do_rec_test $tn $sql $res]
}
foreach {tn setup} {
1 {
if {![file executable $CMD]} { continue }
proc do_rec_test {tn sql res} {
set res [squish [string trim $res]]
set tst [subst -nocommands {
squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
}]
uplevel [list do_test $tn $tst $res]
}
}
2 {
if {[info commands sqlite3_expert_new]==""} { continue }
proc do_rec_test {tn sql res} {
set expert [sqlite3_expert_new db]
$expert sql $sql
$expert analyze
set result [list]
for {set i 0} {$i < [$expert count]} {incr i} {
set idx [string trim [$expert report $i indexes]]
if {$idx==""} {set idx "(no new indexes)"}
lappend result $idx
lappend result [string trim [$expert report $i plan]]
}
$expert destroy
set tst [subst -nocommands {set {} [squish [join {$result}]]}]
uplevel [list do_test $tn $tst [string trim [squish $res]]]
}
}
3 {
if {![file executable $CLI]} { continue }
proc do_rec_test {tn sql res} {
set res [squish [string trim $res]]
set tst [subst -nocommands {
squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
}]
uplevel [list do_test $tn $tst $res]
}
}
} {
eval $setup
do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
SELECT * FROM t1
} {
(no new indexes)
0|0|0|SCAN TABLE t1
}
do_setup_rec_test $tn.2 {
CREATE TABLE t1(a, b, c);
} {
SELECT * FROM t1 WHERE b>?;
} {
CREATE INDEX t1_idx_00000062 ON t1(b);
0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
}
do_setup_rec_test $tn.3 {
CREATE TABLE t1(a, b, c);
} {
SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
} {
CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
}
do_setup_rec_test $tn.4 {
CREATE TABLE t1(a, b, c);
} {
SELECT a FROM t1 ORDER BY b;
} {
CREATE INDEX t1_idx_00000062 ON t1(b);
0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
}
do_setup_rec_test $tn.5 {
CREATE TABLE t1(a, b, c);
} {
SELECT a FROM t1 WHERE a=? ORDER BY b;
} {
CREATE INDEX t1_idx_000123a7 ON t1(a, b);
0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
}
do_setup_rec_test $tn.6 {
CREATE TABLE t1(a, b, c);
} {
SELECT min(a) FROM t1
} {
CREATE INDEX t1_idx_00000061 ON t1(a);
0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
}
do_setup_rec_test $tn.7 {
CREATE TABLE t1(a, b, c);
} {
SELECT * FROM t1 ORDER BY a, b, c;
} {
CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
}
#do_setup_rec_test $tn.1.8 {
# CREATE TABLE t1(a, b, c);
#} {
# SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
#} {
# CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
# 0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
#}
do_setup_rec_test $tn.8.1 {
CREATE TABLE t1(a COLLATE NOCase, b, c);
} {
SELECT * FROM t1 WHERE a=?
} {
CREATE INDEX t1_idx_00000061 ON t1(a);
0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
}
do_setup_rec_test $tn.8.2 {
CREATE TABLE t1(a, b COLLATE nocase, c);
} {
SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
} {
CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
}
# Tables with names that require quotes.
#
do_setup_rec_test $tn.9.1 {
CREATE TABLE "t t"(a, b, c);
} {
SELECT * FROM "t t" WHERE a=?
} {
CREATE INDEX 't t_idx_00000061' ON 't t'(a);
0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?)
}
do_setup_rec_test $tn.9.2 {
CREATE TABLE "t t"(a, b, c);
} {
SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
} {
CREATE INDEX 't t_idx_00000062' ON 't t'(b);
0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
}
# Columns with names that require quotes.
#
do_setup_rec_test $tn.10.1 {
CREATE TABLE t3(a, "b b", c);
} {
SELECT * FROM t3 WHERE "b b" = ?
} {
CREATE INDEX t3_idx_00050c52 ON t3('b b');
0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
}
do_setup_rec_test $tn.10.2 {
CREATE TABLE t3(a, "b b", c);
} {
SELECT * FROM t3 ORDER BY "b b"
} {
CREATE INDEX t3_idx_00050c52 ON t3('b b');
0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
}
# Transitive constraints
#
do_setup_rec_test $tn.11.1 {
CREATE TABLE t5(a, b);
CREATE TABLE t6(c, d);
} {
SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
} {
CREATE INDEX t5_idx_000123a7 ON t5(a, b);
CREATE INDEX t6_idx_00000063 ON t6(c);
0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?)
0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
}
# OR terms.
#
do_setup_rec_test $tn.12.1 {
CREATE TABLE t7(a, b);
} {
SELECT * FROM t7 WHERE a=? OR b=?
} {
CREATE INDEX t7_idx_00000062 ON t7(b);
CREATE INDEX t7_idx_00000061 ON t7(a);
0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?)
0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
}
# rowid terms.
#
do_setup_rec_test $tn.13.1 {
CREATE TABLE t8(a, b);
} {
SELECT * FROM t8 WHERE rowid=?
} {
(no new indexes)
0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
}
do_setup_rec_test $tn.13.2 {
CREATE TABLE t8(a, b);
} {
SELECT * FROM t8 ORDER BY rowid
} {
(no new indexes)
0|0|0|SCAN TABLE t8
}
do_setup_rec_test $tn.13.3 {
CREATE TABLE t8(a, b);
} {
SELECT * FROM t8 WHERE a=? ORDER BY rowid
} {
CREATE INDEX t8_idx_00000061 ON t8(a);
0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
}
# Triggers
#
do_setup_rec_test $tn.14 {
CREATE TABLE t9(a, b, c);
CREATE TABLE t10(a, b, c);
CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
UPDATE t10 SET a=new.a WHERE b = new.b;
END;
} {
INSERT INTO t9 VALUES(?, ?, ?);
} {
CREATE INDEX t10_idx_00000062 ON t10(b);
0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
}
do_setup_rec_test $tn.15 {
CREATE TABLE t1(a, b);
CREATE TABLE t2(c, d);
WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
} {
SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
} {
CREATE INDEX t2_idx_00000064 ON t2(d);
0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?)
0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
}
do_setup_rec_test $tn.16 {
CREATE TABLE t1(a, b);
} {
SELECT * FROM t1 WHERE b IS NOT NULL;
} {
(no new indexes)
0|0|0|SCAN TABLE t1
}
}
proc do_candidates_test {tn sql res} {
set res [squish [string trim $res]]
set expert [sqlite3_expert_new db]
$expert sql $sql
$expert analyze
set candidates [squish [string trim [$expert report 0 candidates]]]
$expert destroy
uplevel [list do_test $tn [list set {} $candidates] $res]
}
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a, b);
CREATE TABLE t2(c, d);
WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
}
do_candidates_test 3.1 {
SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
} {
CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20
CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50
CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20
CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
}
do_candidates_test 3.2 {
SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
} {
CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
}
do_execsql_test 3.2 {
CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50
CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20
CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20
CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
ANALYZE;
SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
} {
t1 t1_idx_00000061 {100 50}
t1 t1_idx_00000062 {100 20}
t1 t1_idx_000123a7 {100 50 17}
t2 t2_idx_00000063 {100 20}
t2 t2_idx_00000064 {100 5}
t2 t2_idx_0001295b {100 20 5}
}
finish_test
File diff suppressed because it is too large Load Diff
-168
View File
@@ -1,168 +0,0 @@
/*
** 2017 April 07
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#include "sqlite3.h"
typedef struct sqlite3expert sqlite3expert;
/*
** Create a new sqlite3expert object.
**
** If successful, a pointer to the new object is returned and (*pzErr) set
** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
** an English-language error message. In this case it is the responsibility
** of the caller to eventually free the error message buffer using
** sqlite3_free().
*/
sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
/*
** Configure an sqlite3expert object.
**
** EXPERT_CONFIG_SAMPLE:
** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
** each candidate index. This involves scanning and sorting the entire
** contents of each user database table once for each candidate index
** associated with the table. For large databases, this can be
** prohibitively slow. This option allows the sqlite3expert object to
** be configured so that sqlite_stat1 data is instead generated based on a
** subset of each table, or so that no sqlite_stat1 data is used at all.
**
** A single integer argument is passed to this option. If the value is less
** than or equal to zero, then no sqlite_stat1 data is generated or used by
** the analysis - indexes are recommended based on the database schema only.
** Or, if the value is 100 or greater, complete sqlite_stat1 data is
** generated for each candidate index (this is the default). Finally, if the
** value falls between 0 and 100, then it represents the percentage of user
** table rows that should be considered when generating sqlite_stat1 data.
**
** Examples:
**
** // Do not generate any sqlite_stat1 data
** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
**
** // Generate sqlite_stat1 data based on 10% of the rows in each table.
** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
*/
int sqlite3_expert_config(sqlite3expert *p, int op, ...);
#define EXPERT_CONFIG_SAMPLE 1 /* int */
/*
** Specify zero or more SQL statements to be included in the analysis.
**
** Buffer zSql must contain zero or more complete SQL statements. This
** function parses all statements contained in the buffer and adds them
** to the internal list of statements to analyze. If successful, SQLITE_OK
** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
** may be set to point to an English language error message. In this case
** the caller is responsible for eventually freeing the error message buffer
** using sqlite3_free().
**
** If an error does occur while processing one of the statements in the
** buffer passed as the second argument, none of the statements in the
** buffer are added to the analysis.
**
** This function must be called before sqlite3_expert_analyze(). If a call
** to this function is made on an sqlite3expert object that has already
** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
** immediately and no statements are added to the analysis.
*/
int sqlite3_expert_sql(
sqlite3expert *p, /* From a successful sqlite3_expert_new() */
const char *zSql, /* SQL statement(s) to add */
char **pzErr /* OUT: Error message (if any) */
);
/*
** This function is called after the sqlite3expert object has been configured
** with all SQL statements using sqlite3_expert_sql() to actually perform
** the analysis. Once this function has been called, it is not possible to
** add further SQL statements to the analysis.
**
** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
** an error occurs, an SQLite error code is returned and (*pzErr) set to
** point to a buffer containing an English language error message. In this
** case it is the responsibility of the caller to eventually free the buffer
** using sqlite3_free().
**
** If an error does occur within this function, the sqlite3expert object
** is no longer useful for any purpose. At that point it is no longer
** possible to add further SQL statements to the object or to re-attempt
** the analysis. The sqlite3expert object must still be freed using a call
** sqlite3_expert_destroy().
*/
int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
/*
** Return the total number of statements loaded using sqlite3_expert_sql().
** The total number of SQL statements may be different from the total number
** to calls to sqlite3_expert_sql().
*/
int sqlite3_expert_count(sqlite3expert*);
/*
** Return a component of the report.
**
** This function is called after sqlite3_expert_analyze() to extract the
** results of the analysis. Each call to this function returns either a
** NULL pointer or a pointer to a buffer containing a nul-terminated string.
** The value passed as the third argument must be one of the EXPERT_REPORT_*
** #define constants defined below.
**
** For some EXPERT_REPORT_* parameters, the buffer returned contains
** information relating to a specific SQL statement. In these cases that
** SQL statement is identified by the value passed as the second argument.
** SQL statements are numbered from 0 in the order in which they are parsed.
** If an out-of-range value (less than zero or equal to or greater than the
** value returned by sqlite3_expert_count()) is passed as the second argument
** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
**
** EXPERT_REPORT_SQL:
** Return the text of SQL statement iStmt.
**
** EXPERT_REPORT_INDEXES:
** Return a buffer containing the CREATE INDEX statements for all recommended
** indexes for statement iStmt. If there are no new recommeded indexes, NULL
** is returned.
**
** EXPERT_REPORT_PLAN:
** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
** iStmt after the proposed indexes have been added to the database schema.
**
** EXPERT_REPORT_CANDIDATES:
** Return a pointer to a buffer containing the CREATE INDEX statements
** for all indexes that were tested (for all SQL statements). The iStmt
** parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
*/
const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
/*
** Values for the third argument passed to sqlite3_expert_report().
*/
#define EXPERT_REPORT_SQL 1
#define EXPERT_REPORT_INDEXES 2
#define EXPERT_REPORT_PLAN 3
#define EXPERT_REPORT_CANDIDATES 4
/*
** Free an (sqlite3expert*) handle and all associated resources. There
** should be one call to this function for each successful call to
** sqlite3-expert_new().
*/
void sqlite3_expert_destroy(sqlite3expert*);
-220
View File
@@ -1,220 +0,0 @@
/*
** 2017 April 07
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#if defined(SQLITE_TEST)
#include "sqlite3expert.h"
#include <assert.h>
#include <string.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Extract an sqlite3* db handle from the object passed as the second
** argument. If successful, set *pDb to point to the db handle and return
** TCL_OK. Otherwise, return TCL_ERROR.
*/
static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
Tcl_CmdInfo info;
if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
return TCL_ERROR;
}
*pDb = *(sqlite3 **)info.objClientData;
return TCL_OK;
}
/*
** Tclcmd: $expert sql SQL
** $expert analyze
** $expert count
** $expert report STMT EREPORT
** $expert destroy
*/
static int SQLITE_TCLAPI testExpertCmd(
void *clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3expert *pExpert = (sqlite3expert*)clientData;
struct Subcmd {
const char *zSub;
int nArg;
const char *zMsg;
} aSub[] = {
{ "sql", 1, "TABLE", }, /* 0 */
{ "analyze", 0, "", }, /* 1 */
{ "count", 0, "", }, /* 2 */
{ "report", 2, "STMT EREPORT", }, /* 3 */
{ "destroy", 0, "", }, /* 4 */
{ 0 }
};
int iSub;
int rc = TCL_OK;
char *zErr = 0;
if( objc<2 ){
Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
return TCL_ERROR;
}
rc = Tcl_GetIndexFromObjStruct(interp,
objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
);
if( rc!=TCL_OK ) return rc;
if( objc!=2+aSub[iSub].nArg ){
Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
return TCL_ERROR;
}
switch( iSub ){
case 0: { /* sql */
char *zArg = Tcl_GetString(objv[2]);
rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
break;
}
case 1: { /* analyze */
rc = sqlite3_expert_analyze(pExpert, &zErr);
break;
}
case 2: { /* count */
int n = sqlite3_expert_count(pExpert);
Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
break;
}
case 3: { /* report */
const char *aEnum[] = {
"sql", "indexes", "plan", "candidates", 0
};
int iEnum;
int iStmt;
const char *zReport;
if( Tcl_GetIntFromObj(interp, objv[2], &iStmt)
|| Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
){
return TCL_ERROR;
}
assert( EXPERT_REPORT_SQL==1 );
assert( EXPERT_REPORT_INDEXES==2 );
assert( EXPERT_REPORT_PLAN==3 );
assert( EXPERT_REPORT_CANDIDATES==4 );
zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
break;
}
default: /* destroy */
assert( iSub==4 );
Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
break;
}
if( rc!=TCL_OK ){
if( zErr ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
}else{
extern const char *sqlite3ErrName(int);
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
}
}
sqlite3_free(zErr);
return rc;
}
static void SQLITE_TCLAPI testExpertDel(void *clientData){
sqlite3expert *pExpert = (sqlite3expert*)clientData;
sqlite3_expert_destroy(pExpert);
}
/*
** sqlite3_expert_new DB
*/
static int SQLITE_TCLAPI test_sqlite3_expert_new(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
static int iCmd = 0;
sqlite3 *db;
char *zCmd = 0;
char *zErr = 0;
sqlite3expert *pExpert;
int rc = TCL_OK;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB");
return TCL_ERROR;
}
if( dbHandleFromObj(interp, objv[1], &db) ){
return TCL_ERROR;
}
zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
if( zCmd==0 ){
Tcl_AppendResult(interp, "out of memory", (char*)0);
return TCL_ERROR;
}
pExpert = sqlite3_expert_new(db, &zErr);
if( pExpert==0 ){
Tcl_AppendResult(interp, zErr, (char*)0);
rc = TCL_ERROR;
}else{
void *p = (void*)pExpert;
Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
}
sqlite3_free(zCmd);
sqlite3_free(zErr);
return rc;
}
#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
int TestExpert_Init(Tcl_Interp *interp){
#ifndef SQLITE_OMIT_VIRTUALTABLE
struct Cmd {
const char *zCmd;
Tcl_ObjCmdProc *xProc;
} aCmd[] = {
{ "sqlite3_expert_new", test_sqlite3_expert_new },
};
int i;
for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
struct Cmd *p = &aCmd[i];
Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
}
#endif
return TCL_OK;
}
#endif
+5 -8
View File
@@ -205,13 +205,13 @@ static int getVarint32(const char *p, int *pi){
*/
/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
static int safe_isspace(char c){
return (c&0x80)==0 ? isspace((unsigned char)c) : 0;
return (c&0x80)==0 ? isspace(c) : 0;
}
static int safe_tolower(char c){
return (c&0x80)==0 ? tolower((unsigned char)c) : c;
return (c&0x80)==0 ? tolower(c) : c;
}
static int safe_isalnum(char c){
return (c&0x80)==0 ? isalnum((unsigned char)c) : 0;
return (c&0x80)==0 ? isalnum(c) : 0;
}
typedef enum DocListType {
@@ -3335,11 +3335,8 @@ int sqlite3Fts1Init(sqlite3 *db){
}
#if !SQLITE_CORE
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fts1_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi){
int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi){
SQLITE_EXTENSION_INIT2(pApi)
return sqlite3Fts1Init(db);
}
+7 -22
View File
@@ -852,14 +852,8 @@ static void fulltext_vtab_destroy(fulltext_vtab *v){
** argv[3] - tokenizer name (optional, a sensible default is provided)
** argv[4..] - passed to tokenizer (optional based on tokenizer)
**/
static int fulltextConnect(
sqlite3 *db,
void *pAux,
int argc,
const char * const *argv,
sqlite3_vtab **ppVTab,
char **pzErr
){
static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv,
sqlite3_vtab **ppVTab){
int rc;
fulltext_vtab *v;
sqlite3_tokenizer_module *m = NULL;
@@ -904,14 +898,8 @@ static int fulltextConnect(
return SQLITE_OK;
}
static int fulltextCreate(
sqlite3 *db,
void *pAux,
int argc,
const char * const *argv,
sqlite3_vtab **ppVTab,
char **pzErr
){
static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv,
sqlite3_vtab **ppVTab){
int rc;
assert( argc>=3 );
@@ -946,7 +934,7 @@ static int fulltextCreate(
"create index %_index on %_term(term, first)");
if( rc!=SQLITE_OK ) return rc;
return fulltextConnect(db, pAux, argc, argv, ppVTab, pzErr);
return fulltextConnect(db, pAux, argc, argv, ppVTab);
}
/* Decide how to handle an SQL query.
@@ -1500,11 +1488,8 @@ int fulltext_init(sqlite3 *db){
}
#if !SQLITE_CORE
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fulltext_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi){
int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi){
SQLITE_EXTENSION_INIT2(pApi)
return fulltext_init(db);
}
+1 -1
View File
@@ -138,7 +138,7 @@ static int simpleNext(
** case-insensitivity.
*/
char ch = c->pCurrent[ii];
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
}
c->zToken[n] = '\0';
*ppToken = c->zToken;
+1 -4
View File
@@ -6844,10 +6844,7 @@ int sqlite3Fts2Init(sqlite3 *db){
}
#if !SQLITE_CORE
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fts2_init(
int sqlite3_extension_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
-2
View File
@@ -30,8 +30,6 @@
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_hash.h"
/*
-3
View File
@@ -30,9 +30,6 @@
#include <stdio.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_tokenizer.h"
/*
+2 -6
View File
@@ -28,7 +28,7 @@
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
SQLITE_EXTENSION_INIT1
#include "fts2_hash.h"
#include "fts2_tokenizer.h"
@@ -99,11 +99,7 @@ static void scalarFunc(
#ifdef SQLITE_TEST
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <tcl.h>
#include <string.h>
/*
-3
View File
@@ -30,9 +30,6 @@
#include <stdio.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_tokenizer.h"
typedef struct simple_tokenizer {
+361 -970
View File
File diff suppressed because it is too large Load Diff
+12 -71
View File
@@ -18,12 +18,6 @@
# define NDEBUG 1
#endif
/* FTS3/FTS4 require virtual tables */
#ifdef SQLITE_OMIT_VIRTUALTABLE
# undef SQLITE_ENABLE_FTS3
# undef SQLITE_ENABLE_FTS4
#endif
/*
** FTS4 is really an extension for FTS3. It is enabled using the
** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all
@@ -38,25 +32,13 @@
/* If not building as part of the core, include sqlite3ext.h. */
#ifndef SQLITE_CORE
# include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
extern const sqlite3_api_routines *sqlite3_api;
#endif
#include "sqlite3.h"
#include "fts3_tokenizer.h"
#include "fts3_hash.h"
/*
** This constant determines the maximum depth of an FTS expression tree
** that the library will create and use. FTS uses recursion to perform
** various operations on the query tree, so the disadvantage of a large
** limit is that it may allow very large queries to use large amounts
** of stack space (perhaps causing a stack overflow).
*/
#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH
# define SQLITE_FTS3_MAX_EXPR_DEPTH 12
#endif
/*
** This constant controls how often segments are merged. Once there are
** FTS3_MERGE_COUNT segments of level N, they are merged into a single
@@ -140,11 +122,6 @@ SQLITE_EXTENSION_INIT3
#ifdef SQLITE_COVERAGE_TEST
# define ALWAYS(x) (1)
# define NEVER(X) (0)
#elif defined(SQLITE_DEBUG)
# define ALWAYS(x) sqlite3Fts3Always((x)!=0)
# define NEVER(x) sqlite3Fts3Never((x)!=0)
int sqlite3Fts3Always(int b);
int sqlite3Fts3Never(int b);
#else
# define ALWAYS(x) (x)
# define NEVER(x) (x)
@@ -203,8 +180,6 @@ typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
typedef struct MatchinfoBuffer MatchinfoBuffer;
/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
@@ -219,25 +194,23 @@ struct Fts3Table {
const char *zName; /* virtual table name */
int nColumn; /* number of named columns in virtual table */
char **azColumn; /* column names. malloced */
u8 *abNotindexed; /* True for 'notindexed' columns */
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
char *zContentTbl; /* content=xxx option, or NULL */
char *zLanguageid; /* languageid=xxx option, or NULL */
int nAutoincrmerge; /* Value configured by 'automerge' */
u8 bAutoincrmerge; /* True if automerge=1 */
u32 nLeafAdd; /* Number of leaf blocks added this trans */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[40];
sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */
sqlite3_stmt *aStmt[37];
char *zReadExprlist;
char *zWriteExprlist;
int nNodeSize; /* Soft limit for node size */
u8 bFts4; /* True for FTS4, false for FTS3 */
u8 bHasStat; /* True if %_stat table exists (2==unknown) */
u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
u8 bDescIdx; /* True if doclists are in reverse order */
u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */
@@ -271,7 +244,6 @@ struct Fts3Table {
int nPendingData; /* Current bytes of pending data */
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
int iPrevLangid; /* Langid of recently inserted document */
int bPrevDelete; /* True if last operation was a delete */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
/* State variables used for validating that the transaction control
@@ -282,12 +254,6 @@ struct Fts3Table {
int inTransaction; /* True after xBegin but before xCommit/xRollback */
int mxSavepoint; /* Largest valid xSavepoint integer */
#endif
#ifdef SQLITE_TEST
/* True to disable the incremental doclist optimization. This is controled
** by special insert command 'test-no-incr-doclist'. */
int bNoIncrDoclist;
#endif
};
/*
@@ -300,7 +266,6 @@ struct Fts3Cursor {
i16 eSearch; /* Search strategy (see below) */
u8 isEof; /* True if at End Of Results */
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
u8 bSeekStmt; /* True if pStmt is a seek */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
int iLangid; /* Language being queried for */
@@ -314,10 +279,11 @@ struct Fts3Cursor {
int eEvalmode; /* An FTS3_EVAL_XX constant */
int nRowAvg; /* Average size of database rows, in pages */
sqlite3_int64 nDoc; /* Documents in table */
i64 iMinDocid; /* Minimum docid to return */
i64 iMaxDocid; /* Maximum docid to return */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
u32 *aMatchinfo; /* Information about most recent match */
int nMatchinfo; /* Number of elements in aMatchinfo[] */
char *zMatchinfo; /* Matchinfo specification */
};
#define FTS3_EVAL_FILTER 0
@@ -343,15 +309,6 @@ struct Fts3Cursor {
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
/*
** The lower 16-bits of the sqlite3_index_info.idxNum value set by
** the xBestIndex() method contains the Fts3Cursor.eSearch value described
** above. The upper 16-bits contain a combination of the following
** bits, used to describe extra constraints on full-text searches.
*/
#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */
#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */
#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */
struct Fts3Doclist {
char *aAll; /* Array containing doclist (or NULL) */
@@ -389,11 +346,6 @@ struct Fts3Phrase {
int bIncr; /* True if doclist is loaded incrementally */
int iDoclistToken;
/* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an
** OR condition. */
char *pOrPoslist;
i64 iOrDocid;
/* Variables below this point are populated by fts3_expr.c when parsing
** a MATCH expression. Everything above is part of the evaluation phase.
*/
@@ -437,9 +389,7 @@ struct Fts3Expr {
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
/* The following are used by the fts3_snippet.c module. */
int iPhrase; /* Index of this phrase in matchinfo() results */
u32 *aMI; /* See above */
u32 *aMI;
};
/*
@@ -471,6 +421,7 @@ int sqlite3Fts3SegReaderPending(
Fts3Table*,int,const char*,int,int,Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
@@ -545,12 +496,7 @@ struct Fts3MultiSegReader {
int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
#define fts3GetVarint32(p, piVal) ( \
(*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \
)
/* fts3.c */
void sqlite3Fts3ErrMsg(char**,const char*,...);
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
@@ -560,7 +506,6 @@ void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@@ -576,11 +521,10 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
char **, int, int, int, const char *, int, Fts3Expr **, char **
char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
@@ -605,11 +549,8 @@ int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
/* fts3_tokenize_vtab.c */
int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
/* fts3_unicode2.c (functions generated by parsing unicode text files) */
#ifndef SQLITE_DISABLE_FTS3_UNICODE
#ifdef SQLITE_ENABLE_FTS4_UNICODE61
int sqlite3FtsUnicodeFold(int, int);
int sqlite3FtsUnicodeIsalnum(int);
int sqlite3FtsUnicodeIsdiacritic(int);
+41 -117
View File
@@ -31,7 +31,6 @@ struct Fts3auxCursor {
Fts3SegFilter filter;
char *zStop;
int nStop; /* Byte-length of string zStop */
int iLangid; /* Language id to query */
int isEof; /* True if cursor is at EOF */
sqlite3_int64 iRowid; /* Current rowid */
@@ -46,8 +45,7 @@ struct Fts3auxCursor {
/*
** Schema of the terms table.
*/
#define FTS3_AUX_SCHEMA \
"CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)"
#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)"
/*
** This function does all the work for both the xConnect and xCreate methods.
@@ -72,29 +70,20 @@ static int fts3auxConnectMethod(
UNUSED_PARAMETER(pUnused);
/* The user should invoke this in one of two forms:
**
** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table);
** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table);
*/
if( argc!=4 && argc!=5 ) goto bad_args;
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
*pzErr = sqlite3_mprintf(
"wrong number of arguments to fts4aux constructor"
);
return SQLITE_ERROR;
}
zDb = argv[1];
nDb = (int)strlen(zDb);
if( argc==5 ){
if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){
zDb = argv[3];
nDb = (int)strlen(zDb);
zFts3 = argv[4];
}else{
goto bad_args;
}
}else{
zFts3 = argv[3];
}
zFts3 = argv[3];
nFts3 = (int)strlen(zFts3);
rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA);
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
@@ -114,10 +103,6 @@ static int fts3auxConnectMethod(
*ppVtab = (sqlite3_vtab *)p;
return SQLITE_OK;
bad_args:
sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor");
return SQLITE_ERROR;
}
/*
@@ -154,8 +139,6 @@ static int fts3auxBestIndexMethod(
int iEq = -1;
int iGe = -1;
int iLe = -1;
int iLangid = -1;
int iNext = 1; /* Next free argvIndex value */
UNUSED_PARAMETER(pVTab);
@@ -167,48 +150,36 @@ static int fts3auxBestIndexMethod(
pInfo->orderByConsumed = 1;
}
/* Search for equality and range constraints on the "term" column.
** And equality constraints on the hidden "languageid" column. */
/* Search for equality and range constraints on the "term" column. */
for(i=0; i<pInfo->nConstraint; i++){
if( pInfo->aConstraint[i].usable ){
if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){
int op = pInfo->aConstraint[i].op;
int iCol = pInfo->aConstraint[i].iColumn;
if( iCol==0 ){
if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
}
if( iCol==4 ){
if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i;
}
if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
}
}
if( iEq>=0 ){
pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT;
pInfo->aConstraintUsage[iEq].argvIndex = iNext++;
pInfo->aConstraintUsage[iEq].argvIndex = 1;
pInfo->estimatedCost = 5;
}else{
pInfo->idxNum = 0;
pInfo->estimatedCost = 20000;
if( iGe>=0 ){
pInfo->idxNum += FTS4AUX_GE_CONSTRAINT;
pInfo->aConstraintUsage[iGe].argvIndex = iNext++;
pInfo->aConstraintUsage[iGe].argvIndex = 1;
pInfo->estimatedCost /= 2;
}
if( iLe>=0 ){
pInfo->idxNum += FTS4AUX_LE_CONSTRAINT;
pInfo->aConstraintUsage[iLe].argvIndex = iNext++;
pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0);
pInfo->estimatedCost /= 2;
}
}
if( iLangid>=0 ){
pInfo->aConstraintUsage[iLangid].argvIndex = iNext++;
pInfo->estimatedCost--;
}
return SQLITE_OK;
}
@@ -368,14 +339,7 @@ static int fts3auxFilterMethod(
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
int rc;
int isScan = 0;
int iLangVal = 0; /* Language id to query */
int iEq = -1; /* Index of term=? value in apVal */
int iGe = -1; /* Index of term>=? value in apVal */
int iLe = -1; /* Index of term<=? value in apVal */
int iLangid = -1; /* Index of languageid=? value in apVal */
int iNext = 0;
int isScan;
UNUSED_PARAMETER(nVal);
UNUSED_PARAMETER(idxStr);
@@ -385,21 +349,7 @@ static int fts3auxFilterMethod(
|| idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT
|| idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT)
);
if( idxNum==FTS4AUX_EQ_CONSTRAINT ){
iEq = iNext++;
}else{
isScan = 1;
if( idxNum & FTS4AUX_GE_CONSTRAINT ){
iGe = iNext++;
}
if( idxNum & FTS4AUX_LE_CONSTRAINT ){
iLe = iNext++;
}
}
if( iNext<nVal ){
iLangid = iNext++;
}
isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT);
/* In case this cursor is being reused, close and zero it. */
testcase(pCsr->filter.zTerm);
@@ -411,35 +361,22 @@ static int fts3auxFilterMethod(
pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
if( iEq>=0 || iGe>=0 ){
if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){
const unsigned char *zStr = sqlite3_value_text(apVal[0]);
assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) );
if( zStr ){
pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
}
}
if( iLe>=0 ){
pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe]));
pCsr->nStop = sqlite3_value_bytes(apVal[iLe]);
if( idxNum&FTS4AUX_LE_CONSTRAINT ){
int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
}
if( iLangid>=0 ){
iLangVal = sqlite3_value_int(apVal[iLangid]);
/* If the user specified a negative value for the languageid, use zero
** instead. This works, as the "languageid=?" constraint will also
** be tested by the VDBE layer. The test will always be false (since
** this module will not return a row with a negative languageid), and
** so the overall query will return zero rows. */
if( iLangVal<0 ) iLangVal = 0;
}
pCsr->iLangid = iLangVal;
rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL,
rc = sqlite3Fts3SegReaderCursor(pFts3, 0, 0, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
);
if( rc==SQLITE_OK ){
@@ -463,37 +400,24 @@ static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
*/
static int fts3auxColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
assert( p->isEof==0 );
switch( iCol ){
case 0: /* term */
sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
break;
case 1: /* col */
if( p->iCol ){
sqlite3_result_int(pCtx, p->iCol-1);
}else{
sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC);
}
break;
case 2: /* documents */
sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc);
break;
case 3: /* occurrences */
sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc);
break;
default: /* languageid */
assert( iCol==4 );
sqlite3_result_int(pCtx, p->iLangid);
break;
if( iCol==0 ){ /* Column "term" */
sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
}else if( iCol==1 ){ /* Column "col" */
if( p->iCol ){
sqlite3_result_int(pContext, p->iCol-1);
}else{
sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC);
}
}else if( iCol==2 ){ /* Column "documents" */
sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc);
}else{ /* Column "occurrences" */
sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc);
}
return SQLITE_OK;
+137 -433
View File
@@ -155,11 +155,6 @@ int sqlite3Fts3OpenTokenizer(
return rc;
}
/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
/*
** Extract the next token from buffer z (length n) using the tokenizer
@@ -185,16 +180,9 @@ static int getNextToken(
int rc;
sqlite3_tokenizer_cursor *pCursor;
Fts3Expr *pRet = 0;
int i = 0;
int nConsumed = 0;
/* Set variable i to the maximum number of bytes of input to tokenize. */
for(i=0; i<n; i++){
if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
if( z[i]=='"' ) break;
}
*pnConsumed = i;
rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
@@ -235,14 +223,13 @@ static int getNextToken(
}
}
*pnConsumed = iEnd;
}else if( i && rc==SQLITE_DONE ){
rc = SQLITE_OK;
nConsumed = iEnd;
}
pModule->xClose(pCursor);
}
*pnConsumed = nConsumed;
*ppExpr = pRet;
return rc;
}
@@ -382,6 +369,12 @@ no_mem:
return SQLITE_NOMEM;
}
/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
/*
** The output variable *ppExpr is populated with an allocated Fts3Expr
** structure, or set to 0 if the end of the input buffer is reached.
@@ -478,6 +471,27 @@ static int getNextNode(
}
}
/* Check for an open bracket. */
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed;
pParse->nNest++;
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){
rc = SQLITE_DONE;
}
*pnConsumed = (int)((zInput - z) + 1 + nConsumed);
return rc;
}
/* Check for a close bracket. */
if( *zInput==')' ){
pParse->nNest--;
*pnConsumed = (int)((zInput - z) + 1);
return SQLITE_DONE;
}
}
/* See if we are dealing with a quoted phrase. If this is the case, then
** search for the closing quote and pass the whole string to getNextString()
** for processing. This is easy to do, as fts3 has no syntax for escaping
@@ -492,21 +506,6 @@ static int getNextNode(
return getNextString(pParse, &zInput[1], ii-1, ppExpr);
}
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed = 0;
pParse->nNest++;
rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; }
*pnConsumed = (int)(zInput - z) + 1 + nConsumed;
return rc;
}else if( *zInput==')' ){
pParse->nNest--;
*pnConsumed = (int)((zInput - z) + 1);
*ppExpr = 0;
return SQLITE_DONE;
}
}
/* If control flows to this point, this must be a regular token, or
** the end of the input. Read a regular token using the sqlite3_tokenizer
@@ -625,100 +624,94 @@ static int fts3ExprParse(
while( rc==SQLITE_OK ){
Fts3Expr *p = 0;
int nByte = 0;
rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
assert( nByte>0 || (rc!=SQLITE_OK && p==0) );
if( rc==SQLITE_OK ){
if( p ){
int isPhrase;
int isPhrase;
if( !sqlite3_fts3_enable_parentheses
&& p->eType==FTSQUERY_PHRASE && pParse->isNot
){
/* Create an implicit NOT operator. */
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
if( !sqlite3_fts3_enable_parentheses
&& p->eType==FTSQUERY_PHRASE && pParse->isNot
){
/* Create an implicit NOT operator. */
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
if( pNotBranch ){
pNot->pLeft = pNotBranch;
}
pNotBranch = pNot;
p = pPrev;
}else{
int eType = p->eType;
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
/* The isRequirePhrase variable is set to true if a phrase or
** an expression contained in parenthesis is required. If a
** binary operator (AND, OR, NOT or NEAR) is encounted when
** isRequirePhrase is set, this is a syntax error.
*/
if( !isPhrase && isRequirePhrase ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase && !isRequirePhrase ){
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
p->pParent = pNot;
if( pNotBranch ){
pNot->pLeft = pNotBranch;
pNotBranch->pParent = pNot;
}
pNotBranch = pNot;
p = pPrev;
}else{
int eType = p->eType;
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
}
/* The isRequirePhrase variable is set to true if a phrase or
** an expression contained in parenthesis is required. If a
** binary operator (AND, OR, NOT or NEAR) is encounted when
** isRequirePhrase is set, this is a syntax error.
*/
if( !isPhrase && isRequirePhrase ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase && !isRequirePhrase ){
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
}
/* This test catches attempts to make either operand of a NEAR
** operator something other than a phrase. For example, either of
** the following:
**
** (bracketed expression) NEAR phrase
** phrase NEAR (bracketed expression)
**
** Return an error in either case.
*/
if( pPrev && (
/* This test catches attempts to make either operand of a NEAR
** operator something other than a phrase. For example, either of
** the following:
**
** (bracketed expression) NEAR phrase
** phrase NEAR (bracketed expression)
**
** Return an error in either case.
*/
if( pPrev && (
(eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
|| (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
)){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase ){
if( pRet ){
assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
pPrev->pRight = p;
p->pParent = pPrev;
}else{
pRet = p;
}
}else{
insertBinaryOperator(&pRet, pPrev, p);
}
isRequirePhrase = !isPhrase;
)){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
pPrev = p;
if( isPhrase ){
if( pRet ){
assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
pPrev->pRight = p;
p->pParent = pPrev;
}else{
pRet = p;
}
}else{
insertBinaryOperator(&pRet, pPrev, p);
}
isRequirePhrase = !isPhrase;
}
assert( nByte>0 );
}
assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
nIn -= nByte;
zIn += nByte;
pPrev = p;
}
if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
@@ -736,7 +729,6 @@ static int fts3ExprParse(
pIter = pIter->pLeft;
}
pIter->pLeft = pRet;
pRet->pParent = pIter;
pRet = pNotBranch;
}
}
@@ -753,249 +745,6 @@ exprparse_out:
return rc;
}
/*
** Return SQLITE_ERROR if the maximum depth of the expression tree passed
** as the only argument is more than nMaxDepth.
*/
static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){
int rc = SQLITE_OK;
if( p ){
if( nMaxDepth<0 ){
rc = SQLITE_TOOBIG;
}else{
rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1);
if( rc==SQLITE_OK ){
rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1);
}
}
}
return rc;
}
/*
** This function attempts to transform the expression tree at (*pp) to
** an equivalent but more balanced form. The tree is modified in place.
** If successful, SQLITE_OK is returned and (*pp) set to point to the
** new root expression node.
**
** nMaxDepth is the maximum allowable depth of the balanced sub-tree.
**
** Otherwise, if an error occurs, an SQLite error code is returned and
** expression (*pp) freed.
*/
static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){
int rc = SQLITE_OK; /* Return code */
Fts3Expr *pRoot = *pp; /* Initial root node */
Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */
int eType = pRoot->eType; /* Type of node in this tree */
if( nMaxDepth==0 ){
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK ){
if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
Fts3Expr **apLeaf;
apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
if( 0==apLeaf ){
rc = SQLITE_NOMEM;
}else{
memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
}
if( rc==SQLITE_OK ){
int i;
Fts3Expr *p;
/* Set $p to point to the left-most leaf in the tree of eType nodes. */
for(p=pRoot; p->eType==eType; p=p->pLeft){
assert( p->pParent==0 || p->pParent->pLeft==p );
assert( p->pLeft && p->pRight );
}
/* This loop runs once for each leaf in the tree of eType nodes. */
while( 1 ){
int iLvl;
Fts3Expr *pParent = p->pParent; /* Current parent of p */
assert( pParent==0 || pParent->pLeft==p );
p->pParent = 0;
if( pParent ){
pParent->pLeft = 0;
}else{
pRoot = 0;
}
rc = fts3ExprBalance(&p, nMaxDepth-1);
if( rc!=SQLITE_OK ) break;
for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
if( apLeaf[iLvl]==0 ){
apLeaf[iLvl] = p;
p = 0;
}else{
assert( pFree );
pFree->pLeft = apLeaf[iLvl];
pFree->pRight = p;
pFree->pLeft->pParent = pFree;
pFree->pRight->pParent = pFree;
p = pFree;
pFree = pFree->pParent;
p->pParent = 0;
apLeaf[iLvl] = 0;
}
}
if( p ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_TOOBIG;
break;
}
/* If that was the last leaf node, break out of the loop */
if( pParent==0 ) break;
/* Set $p to point to the next leaf in the tree of eType nodes */
for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
/* Remove pParent from the original tree. */
assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
pParent->pRight->pParent = pParent->pParent;
if( pParent->pParent ){
pParent->pParent->pLeft = pParent->pRight;
}else{
assert( pParent==pRoot );
pRoot = pParent->pRight;
}
/* Link pParent into the free node list. It will be used as an
** internal node of the new tree. */
pParent->pParent = pFree;
pFree = pParent;
}
if( rc==SQLITE_OK ){
p = 0;
for(i=0; i<nMaxDepth; i++){
if( apLeaf[i] ){
if( p==0 ){
p = apLeaf[i];
p->pParent = 0;
}else{
assert( pFree!=0 );
pFree->pRight = p;
pFree->pLeft = apLeaf[i];
pFree->pLeft->pParent = pFree;
pFree->pRight->pParent = pFree;
p = pFree;
pFree = pFree->pParent;
p->pParent = 0;
}
}
}
pRoot = p;
}else{
/* An error occurred. Delete the contents of the apLeaf[] array
** and pFree list. Everything else is cleaned up by the call to
** sqlite3Fts3ExprFree(pRoot) below. */
Fts3Expr *pDel;
for(i=0; i<nMaxDepth; i++){
sqlite3Fts3ExprFree(apLeaf[i]);
}
while( (pDel=pFree)!=0 ){
pFree = pDel->pParent;
sqlite3_free(pDel);
}
}
assert( pFree==0 );
sqlite3_free( apLeaf );
}
}else if( eType==FTSQUERY_NOT ){
Fts3Expr *pLeft = pRoot->pLeft;
Fts3Expr *pRight = pRoot->pRight;
pRoot->pLeft = 0;
pRoot->pRight = 0;
pLeft->pParent = 0;
pRight->pParent = 0;
rc = fts3ExprBalance(&pLeft, nMaxDepth-1);
if( rc==SQLITE_OK ){
rc = fts3ExprBalance(&pRight, nMaxDepth-1);
}
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRight);
sqlite3Fts3ExprFree(pLeft);
}else{
assert( pLeft && pRight );
pRoot->pLeft = pLeft;
pLeft->pParent = pRoot;
pRoot->pRight = pRight;
pRight->pParent = pRoot;
}
}
}
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRoot);
pRoot = 0;
}
*pp = pRoot;
return rc;
}
/*
** This function is similar to sqlite3Fts3ExprParse(), with the following
** differences:
**
** 1. It does not do expression rebalancing.
** 2. It does not check that the expression does not exceed the
** maximum allowable depth.
** 3. Even if it fails, *ppExpr may still be set to point to an
** expression tree. It should be deleted using sqlite3Fts3ExprFree()
** in this case.
*/
static int fts3ExprParseUnbalanced(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
int iLangid, /* Language id for tokenizer */
char **azCol, /* Array of column names for fts3 table */
int bFts4, /* True to allow FTS4-only syntax */
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr /* OUT: Parsed query structure */
){
int nParsed;
int rc;
ParseContext sParse;
memset(&sParse, 0, sizeof(ParseContext));
sParse.pTokenizer = pTokenizer;
sParse.iLangid = iLangid;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
assert( rc==SQLITE_OK || *ppExpr==0 );
/* Check for mismatched parenthesis */
if( rc==SQLITE_OK && sParse.nNest ){
rc = SQLITE_ERROR;
}
return rc;
}
/*
** Parameters z and n contain a pointer to and length of a buffer containing
** an fts3 query expression, respectively. This function attempts to parse the
@@ -1028,74 +777,49 @@ int sqlite3Fts3ExprParse(
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr, /* OUT: Parsed query structure */
char **pzErr /* OUT: Error message (sqlite3_malloc) */
Fts3Expr **ppExpr /* OUT: Parsed query structure */
){
int rc = fts3ExprParseUnbalanced(
pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr
);
/* Rebalance the expression. And check that its depth does not exceed
** SQLITE_FTS3_MAX_EXPR_DEPTH. */
if( rc==SQLITE_OK && *ppExpr ){
rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
if( rc==SQLITE_OK ){
rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
}
}
int nParsed;
int rc;
ParseContext sParse;
if( rc!=SQLITE_OK ){
memset(&sParse, 0, sizeof(ParseContext));
sParse.pTokenizer = pTokenizer;
sParse.iLangid = iLangid;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
/* Check for mismatched parenthesis */
if( rc==SQLITE_OK && sParse.nNest ){
rc = SQLITE_ERROR;
sqlite3Fts3ExprFree(*ppExpr);
*ppExpr = 0;
if( rc==SQLITE_TOOBIG ){
sqlite3Fts3ErrMsg(pzErr,
"FTS expression tree is too large (maximum depth %d)",
SQLITE_FTS3_MAX_EXPR_DEPTH
);
rc = SQLITE_ERROR;
}else if( rc==SQLITE_ERROR ){
sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z);
}
}
return rc;
}
/*
** Free a single node of an expression tree.
*/
static void fts3FreeExprNode(Fts3Expr *p){
assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
sqlite3_free(p->aMI);
sqlite3_free(p);
}
/*
** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
**
** This function would be simpler if it recursively called itself. But
** that would mean passing a sufficiently large expression to ExprParse()
** could cause a stack overflow.
*/
void sqlite3Fts3ExprFree(Fts3Expr *pDel){
Fts3Expr *p;
assert( pDel==0 || pDel->pParent==0 );
for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){
assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft );
}
while( p ){
Fts3Expr *pParent = p->pParent;
fts3FreeExprNode(p);
if( pParent && p==pParent->pLeft && pParent->pRight ){
p = pParent->pRight;
while( p && (p->pLeft || p->pRight) ){
assert( p==p->pParent->pRight || p==p->pParent->pLeft );
p = (p->pLeft ? p->pLeft : p->pRight);
}
}else{
p = pParent;
}
void sqlite3Fts3ExprFree(Fts3Expr *p){
if( p ){
assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight);
sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
sqlite3_free(p->aMI);
sqlite3_free(p);
}
}
@@ -1147,9 +871,6 @@ static int queryTestTokenizer(
** the returned expression text and then freed using sqlite3_free().
*/
static char *exprToString(Fts3Expr *pExpr, char *zBuf){
if( pExpr==0 ){
return sqlite3_mprintf("");
}
switch( pExpr->eType ){
case FTSQUERY_PHRASE: {
Fts3Phrase *pPhrase = pExpr->pPhrase;
@@ -1257,21 +978,10 @@ static void fts3ExprTest(
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
if( sqlite3_user_data(context) ){
char *zDummy = 0;
rc = sqlite3Fts3ExprParse(
pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
);
assert( rc==SQLITE_OK || pExpr==0 );
sqlite3_free(zDummy);
}else{
rc = fts3ExprParseUnbalanced(
pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
}
rc = sqlite3Fts3ExprParse(
pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3Fts3ExprFree(pExpr);
sqlite3_result_error(context, "Error parsing expression", -1);
}else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
sqlite3_result_error_nomem(context);
@@ -1294,15 +1004,9 @@ exprtest_out:
** with database connection db.
*/
int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
int rc = sqlite3_create_function(
return sqlite3_create_function(
db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "fts3_exprtest_rebalance",
-1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
);
}
return rc;
}
#endif
+2 -2
View File
@@ -96,13 +96,13 @@ void sqlite3Fts3HashClear(Fts3Hash *pH){
*/
static int fts3StrHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
unsigned h = 0;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return (int)(h & 0x7fffffff);
return h & 0x7fffffff;
}
static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
+6 -7
View File
@@ -240,13 +240,12 @@ static int icuNext(
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
0, /* xLanguageid */
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
};
/*
+30 -46
View File
@@ -183,7 +183,7 @@ static int isVowel(const char *z){
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of a consonant followed by a vowel.
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
@@ -403,14 +403,12 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
/* Step 2 */
switch( z[1] ){
case 'a':
if( !stem(&z, "lanoita", "ate", m_gt_0) ){
stem(&z, "lanoit", "tion", m_gt_0);
}
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
if( !stem(&z, "icne", "ence", m_gt_0) ){
stem(&z, "icna", "ance", m_gt_0);
}
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
@@ -419,54 +417,43 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
if( !stem(&z, "ilb", "ble", m_gt_0)
&& !stem(&z, "illa", "al", m_gt_0)
&& !stem(&z, "iltne", "ent", m_gt_0)
&& !stem(&z, "ile", "e", m_gt_0)
){
stem(&z, "ilsuo", "ous", m_gt_0);
}
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
if( !stem(&z, "noitazi", "ize", m_gt_0)
&& !stem(&z, "noita", "ate", m_gt_0)
){
stem(&z, "rota", "ate", m_gt_0);
}
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
if( !stem(&z, "msila", "al", m_gt_0)
&& !stem(&z, "ssenevi", "ive", m_gt_0)
&& !stem(&z, "ssenluf", "ful", m_gt_0)
){
stem(&z, "ssensuo", "ous", m_gt_0);
}
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
if( !stem(&z, "itila", "al", m_gt_0)
&& !stem(&z, "itivi", "ive", m_gt_0)
){
stem(&z, "itilib", "ble", m_gt_0);
}
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
if( !stem(&z, "etaci", "ic", m_gt_0)
&& !stem(&z, "evita", "", m_gt_0)
){
stem(&z, "ezila", "al", m_gt_0);
}
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
if( !stem(&z, "laci", "ic", m_gt_0) ){
stem(&z, "luf", "", m_gt_0);
}
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
@@ -507,11 +494,9 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
z += 3;
}
}else if( z[2]=='e' ){
if( !stem(&z, "tneme", "", m_gt_1)
&& !stem(&z, "tnem", "", m_gt_1)
){
stem(&z, "tne", "", m_gt_1);
}
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
@@ -530,9 +515,8 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
}
break;
case 't':
if( !stem(&z, "eta", "", m_gt_1) ){
stem(&z, "iti", "", m_gt_1);
}
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+78 -264
View File
@@ -27,8 +27,6 @@
#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
#define FTS3_MATCHINFO_LCS 's' /* nCol values */
#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */
#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */
/*
** The default value for the second argument to matchinfo().
@@ -90,22 +88,9 @@ struct MatchInfo {
int nCol; /* Number of columns in table */
int nPhrase; /* Number of matchable phrases in query */
sqlite3_int64 nDoc; /* Number of docs in database */
char flag;
u32 *aMatchinfo; /* Pre-allocated buffer */
};
/*
** An instance of this structure is used to manage a pair of buffers, each
** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
** for details.
*/
struct MatchinfoBuffer {
u8 aRef[3];
int nElem;
int bGlobal; /* Set if global data is loaded */
char *zMatchinfo;
u32 aMatchinfo[1];
};
/*
@@ -121,97 +106,6 @@ struct StrBuffer {
};
/*************************************************************************
** Start of MatchinfoBuffer code.
*/
/*
** Allocate a two-slot MatchinfoBuffer object.
*/
static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
MatchinfoBuffer *pRet;
int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
int nStr = (int)strlen(zMatchinfo);
pRet = sqlite3_malloc(nByte + nStr+1);
if( pRet ){
memset(pRet, 0, nByte);
pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
pRet->nElem = nElem;
pRet->zMatchinfo = ((char*)pRet) + nByte;
memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
pRet->aRef[0] = 1;
}
return pRet;
}
static void fts3MIBufferFree(void *p){
MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
assert( (u32*)p==&pBuf->aMatchinfo[1]
|| (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2]
);
if( (u32*)p==&pBuf->aMatchinfo[1] ){
pBuf->aRef[1] = 0;
}else{
pBuf->aRef[2] = 0;
}
if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
sqlite3_free(pBuf);
}
}
static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
void (*xRet)(void*) = 0;
u32 *aOut = 0;
if( p->aRef[1]==0 ){
p->aRef[1] = 1;
aOut = &p->aMatchinfo[1];
xRet = fts3MIBufferFree;
}
else if( p->aRef[2]==0 ){
p->aRef[2] = 1;
aOut = &p->aMatchinfo[p->nElem+2];
xRet = fts3MIBufferFree;
}else{
aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
if( aOut ){
xRet = sqlite3_free;
if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
}
*paOut = aOut;
return xRet;
}
static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
p->bGlobal = 1;
memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
/*
** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
*/
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
if( p ){
assert( p->aRef[0]==1 );
p->aRef[0] = 0;
if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
sqlite3_free(p);
}
}
}
/*
** End of MatchinfoBuffer code.
*************************************************************************/
/*
** This function is used to help iterate through a position-list. A position
** list is a list of unique integers, sorted from smallest to largest. Each
@@ -234,7 +128,7 @@ void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
*/
static void fts3GetDeltaPosition(char **pp, int *piPos){
int iVal;
*pp += fts3GetVarint32(*pp, &iVal);
*pp += sqlite3Fts3GetVarint32(*pp, &iVal);
*piPos += (iVal-2);
}
@@ -248,7 +142,7 @@ static int fts3ExprIterate2(
void *pCtx /* Second argument to pass to callback */
){
int rc; /* Return code */
int eType = pExpr->eType; /* Type of expression node pExpr */
int eType = pExpr->eType; /* Type of expression node pExpr */
if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
@@ -282,7 +176,6 @@ static int fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
@@ -327,7 +220,8 @@ static int fts3ExprLoadDoclists(
static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
(*(int *)ctx)++;
pExpr->iPhrase = iPhrase;
UNUSED_PARAMETER(pExpr);
UNUSED_PARAMETER(iPhrase);
return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
@@ -548,39 +442,37 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
if( rc==SQLITE_OK ){
(void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
/* Set the *pmSeen output variable. */
for(i=0; i<nList; i++){
if( sIter.aPhrase[i].pHead ){
*pmSeen |= (u64)1 << i;
}
/* Set the *pmSeen output variable. */
for(i=0; i<nList; i++){
if( sIter.aPhrase[i].pHead ){
*pmSeen |= (u64)1 << i;
}
/* Loop through all candidate snippets. Store the best snippet in
** *pFragment. Store its associated 'score' in iBestScore.
*/
pFragment->iCol = iCol;
while( !fts3SnippetNextCandidate(&sIter) ){
int iPos;
int iScore;
u64 mCover;
u64 mHighlite;
fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite);
assert( iScore>=0 );
if( iScore>iBestScore ){
pFragment->iPos = iPos;
pFragment->hlmask = mHighlite;
pFragment->covered = mCover;
iBestScore = iScore;
}
}
*piScore = iBestScore;
}
/* Loop through all candidate snippets. Store the best snippet in
** *pFragment. Store its associated 'score' in iBestScore.
*/
pFragment->iCol = iCol;
while( !fts3SnippetNextCandidate(&sIter) ){
int iPos;
int iScore;
u64 mCover;
u64 mHighlight;
fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight);
assert( iScore>=0 );
if( iScore>iBestScore ){
pFragment->iPos = iPos;
pFragment->hlmask = mHighlight;
pFragment->covered = mCover;
iBestScore = iScore;
}
}
sqlite3_free(sIter.aPhrase);
return rc;
*piScore = iBestScore;
return SQLITE_OK;
}
@@ -612,7 +504,6 @@ static int fts3StringAppend(
pStr->z = zNew;
pStr->nAlloc = nAlloc;
}
assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) );
/* Append the data to the string buffer. */
memcpy(&pStr->z[pStr->n], zAppend, nAppend);
@@ -788,12 +679,8 @@ static int fts3SnippetText(
** required. They are required if (a) this is not the first fragment,
** or (b) this fragment does not begin at position 0 of its column.
*/
if( rc==SQLITE_OK ){
if( iPos>0 || iFragment>0 ){
rc = fts3StringAppend(pOut, zEllipsis, -1);
}else if( iBegin ){
rc = fts3StringAppend(pOut, zDoc, iBegin);
}
if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){
rc = fts3StringAppend(pOut, zEllipsis, -1);
}
if( rc!=SQLITE_OK || iCurrent<iPos ) continue;
}
@@ -849,60 +736,6 @@ static int fts3ColumnlistCount(char **ppCollist){
return nEntry;
}
/*
** This function gathers 'y' or 'b' data for a single phrase.
*/
static void fts3ExprLHits(
Fts3Expr *pExpr, /* Phrase expression node */
MatchInfo *p /* Matchinfo context */
){
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int iStart;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter = pPhrase->doclist.pList;
int iCol = 0;
assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
if( p->flag==FTS3_MATCHINFO_LHITS ){
iStart = pExpr->iPhrase * p->nCol;
}else{
iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
}
while( 1 ){
int nHit = fts3ColumnlistCount(&pIter);
if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
if( p->flag==FTS3_MATCHINFO_LHITS ){
p->aMatchinfo[iStart + iCol] = (u32)nHit;
}else if( nHit ){
p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
}
}
assert( *pIter==0x00 || *pIter==0x01 );
if( *pIter!=0x01 ) break;
pIter++;
pIter += fts3GetVarint32(pIter, &iCol);
}
}
/*
** Gather the results for matchinfo directives 'y' and 'b'.
*/
static void fts3ExprLHitGather(
Fts3Expr *pExpr,
MatchInfo *p
){
assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
if( pExpr->pLeft ){
fts3ExprLHitGather(pExpr->pLeft, p);
fts3ExprLHitGather(pExpr->pRight, p);
}else{
fts3ExprLHits(pExpr, p);
}
}
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
@@ -981,12 +814,10 @@ static int fts3MatchinfoCheck(
|| (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
|| (cArg==FTS3_MATCHINFO_LCS)
|| (cArg==FTS3_MATCHINFO_HITS)
|| (cArg==FTS3_MATCHINFO_LHITS)
|| (cArg==FTS3_MATCHINFO_LHITS_BM)
){
return SQLITE_OK;
}
sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg);
*pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
return SQLITE_ERROR;
}
@@ -1006,14 +837,6 @@ static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
nVal = pInfo->nCol;
break;
case FTS3_MATCHINFO_LHITS:
nVal = pInfo->nCol * pInfo->nPhrase;
break;
case FTS3_MATCHINFO_LHITS_BM:
nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32);
break;
default:
assert( cArg==FTS3_MATCHINFO_HITS );
nVal = pInfo->nCol * pInfo->nPhrase * 3;
@@ -1208,7 +1031,7 @@ static int fts3MatchinfoValues(
sqlite3_stmt *pSelect = 0;
for(i=0; rc==SQLITE_OK && zArg[i]; i++){
pInfo->flag = zArg[i];
switch( zArg[i] ){
case FTS3_MATCHINFO_NPHRASE:
if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
@@ -1268,14 +1091,6 @@ static int fts3MatchinfoValues(
}
break;
case FTS3_MATCHINFO_LHITS_BM:
case FTS3_MATCHINFO_LHITS: {
int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
fts3ExprLHitGather(pCsr->pExpr, pInfo);
break;
}
default: {
Fts3Expr *pExpr;
assert( zArg[i]==FTS3_MATCHINFO_HITS );
@@ -1288,7 +1103,6 @@ static int fts3MatchinfoValues(
if( rc!=SQLITE_OK ) break;
}
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
sqlite3Fts3EvalTestDeferred(pCsr, &rc);
if( rc!=SQLITE_OK ) break;
}
(void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
@@ -1308,8 +1122,7 @@ static int fts3MatchinfoValues(
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static void fts3GetMatchinfo(
sqlite3_context *pCtx, /* Return results here */
static int fts3GetMatchinfo(
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
@@ -1318,9 +1131,6 @@ static void fts3GetMatchinfo(
int rc = SQLITE_OK;
int bGlobal = 0; /* Collect 'global' stats as well as local */
u32 *aOut = 0;
void (*xDestroyOut)(void*) = 0;
memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
@@ -1328,18 +1138,21 @@ static void fts3GetMatchinfo(
/* If there is cached matchinfo() data, but the format string for the
** cache does not match the format string for this request, discard
** the cached data. */
if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
pCsr->pMIBuffer = 0;
if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
assert( pCsr->aMatchinfo );
sqlite3_free(pCsr->aMatchinfo);
pCsr->zMatchinfo = 0;
pCsr->aMatchinfo = 0;
}
/* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
/* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
if( pCsr->pMIBuffer==0 ){
if( pCsr->aMatchinfo==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
int nArg; /* Bytes in zArg */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
@@ -1348,46 +1161,30 @@ static void fts3GetMatchinfo(
/* Determine the number of integers in the buffer returned by this call. */
for(i=0; zArg[i]; i++){
char *zErr = 0;
if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
return;
}
nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
}
/* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
nArg = (int)strlen(zArg);
pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
pCsr->nMatchinfo = nMatchinfo;
memcpy(pCsr->zMatchinfo, zArg, nArg+1);
memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
pCsr->isMatchinfoNeeded = 1;
bGlobal = 1;
}
if( rc==SQLITE_OK ){
xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
if( xDestroyOut==0 ){
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
sInfo.aMatchinfo = aOut;
sInfo.nPhrase = pCsr->nPhrase;
sInfo.aMatchinfo = pCsr->aMatchinfo;
sInfo.nPhrase = pCsr->nPhrase;
if( pCsr->isMatchinfoNeeded ){
rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
if( bGlobal ){
fts3MIBufferSetGlobal(pCsr->pMIBuffer);
}
pCsr->isMatchinfoNeeded = 0;
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
if( xDestroyOut ) xDestroyOut(aOut);
}else{
int n = pCsr->pMIBuffer->nElem * sizeof(u32);
sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
}
return rc;
}
/*
@@ -1449,7 +1246,7 @@ void sqlite3Fts3Snippet(
*/
for(iRead=0; iRead<pTab->nColumn; iRead++){
SnippetFragment sF = {0, 0, 0, 0};
int iS = 0;
int iS;
if( iCol>=0 && iRead!=iCol ) continue;
/* Find the best snippet of nFToken tokens in column iRead. */
@@ -1593,7 +1390,7 @@ void sqlite3Fts3Offsets(
*/
sCtx.iCol = iCol;
sCtx.iTerm = 0;
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
/* Retreive the text stored in column iCol. If an SQL NULL is stored
** in column iCol, jump immediately to the next iteration of the loop.
@@ -1685,9 +1482,19 @@ void sqlite3Fts3Matchinfo(
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc;
int i;
const char *zFormat;
if( zArg ){
for(i=0; zArg[i]; i++){
char *zErr = 0;
if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
sqlite3_result_error(pContext, zErr, -1);
sqlite3_free(zErr);
return;
}
}
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
@@ -1696,10 +1503,17 @@ void sqlite3Fts3Matchinfo(
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}
/* Retrieve matchinfo() data. */
rc = fts3GetMatchinfo(pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
/* Retrieve matchinfo() data. */
fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
int n = pCsr->nMatchinfo * sizeof(u32);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
}
}
+1 -1
View File
@@ -81,7 +81,7 @@ static int fts3termConnectMethod(
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
sqlite3Fts3ErrMsg(pzErr,
*pzErr = sqlite3_mprintf(
"wrong number of arguments to fts4term constructor"
);
return SQLITE_ERROR;
+5 -62
View File
@@ -18,14 +18,7 @@
** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include <tcl.h>
#include <string.h>
#include <assert.h>
@@ -150,7 +143,7 @@ static int nm_match_count(
/*
** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
*/
static int SQLITE_TCLAPI fts3_near_match_cmd(
static int fts3_near_match_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -285,7 +278,7 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
** # Restore initial incr-load settings:
** eval fts3_configure_incr_load $cfg
*/
static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
static int fts3_configure_incr_load_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -465,7 +458,7 @@ static int testTokenizerNext(
if( pCsr->iLangid & 0x00000001 ){
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
}else{
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]);
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
}
pCsr->iToken++;
pCsr->iInput = (int)(p - pCsr->aInput);
@@ -495,7 +488,7 @@ static int testTokenizerLanguage(
}
#endif
static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
static int fts3_test_tokenizer_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
@@ -524,52 +517,6 @@ static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
return TCL_OK;
}
static int SQLITE_TCLAPI fts3_test_varint_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
#ifdef SQLITE_ENABLE_FTS3
char aBuf[24];
int rc;
Tcl_WideInt w;
sqlite3_int64 w2;
int nByte, nByte2;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
return TCL_ERROR;
}
rc = Tcl_GetWideIntFromObj(interp, objv[1], &w);
if( rc!=TCL_OK ) return rc;
nByte = sqlite3Fts3PutVarint(aBuf, w);
nByte2 = sqlite3Fts3GetVarint(aBuf, &w2);
if( w!=w2 || nByte!=nByte2 ){
char *zErr = sqlite3_mprintf("error testing %lld", w);
Tcl_ResetResult(interp);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
if( w<=2147483647 && w>=0 ){
int i;
nByte2 = fts3GetVarint32(aBuf, &i);
if( (int)w!=i || nByte!=nByte2 ){
char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w);
Tcl_ResetResult(interp);
Tcl_AppendResult(interp, zErr, 0);
return TCL_ERROR;
}
}
#endif
UNUSED_PARAMETER(clientData);
return TCL_OK;
}
/*
** End of tokenizer code.
**************************************************************************/
@@ -582,10 +529,6 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(
interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
);
Tcl_CreateObjCommand(
interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
);
return TCL_OK;
}
#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
-454
View File
@@ -1,454 +0,0 @@
/*
** 2013 Apr 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains code for the "fts3tokenize" virtual table module.
** An fts3tokenize virtual table is created as follows:
**
** CREATE VIRTUAL TABLE <tbl> USING fts3tokenize(
** <tokenizer-name>, <arg-1>, ...
** );
**
** The table created has the following schema:
**
** CREATE TABLE <tbl>(input, token, start, end, position)
**
** When queried, the query must include a WHERE clause of type:
**
** input = <string>
**
** The virtual table module tokenizes this <string>, using the FTS3
** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE
** statement and returns one row for each token in the result. With
** fields set as follows:
**
** input: Always set to a copy of <string>
** token: A token from the input.
** start: Byte offset of the token within the input <string>.
** end: Byte offset of the byte immediately following the end of the
** token within the input string.
** pos: Token offset of token within input.
**
*/
#include "fts3Int.h"
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include <string.h>
#include <assert.h>
typedef struct Fts3tokTable Fts3tokTable;
typedef struct Fts3tokCursor Fts3tokCursor;
/*
** Virtual table structure.
*/
struct Fts3tokTable {
sqlite3_vtab base; /* Base class used by SQLite core */
const sqlite3_tokenizer_module *pMod;
sqlite3_tokenizer *pTok;
};
/*
** Virtual table cursor structure.
*/
struct Fts3tokCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
char *zInput; /* Input string */
sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */
int iRowid; /* Current 'rowid' value */
const char *zToken; /* Current 'token' value */
int nToken; /* Size of zToken in bytes */
int iStart; /* Current 'start' value */
int iEnd; /* Current 'end' value */
int iPos; /* Current 'pos' value */
};
/*
** Query FTS for the tokenizer implementation named zName.
*/
static int fts3tokQueryTokenizer(
Fts3Hash *pHash,
const char *zName,
const sqlite3_tokenizer_module **pp,
char **pzErr
){
sqlite3_tokenizer_module *p;
int nName = (int)strlen(zName);
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName);
return SQLITE_ERROR;
}
*pp = p;
return SQLITE_OK;
}
/*
** The second argument, argv[], is an array of pointers to nul-terminated
** strings. This function makes a copy of the array and strings into a
** single block of memory. It then dequotes any of the strings that appear
** to be quoted.
**
** If successful, output parameter *pazDequote is set to point at the
** array of dequoted strings and SQLITE_OK is returned. The caller is
** responsible for eventually calling sqlite3_free() to free the array
** in this case. Or, if an error occurs, an SQLite error code is returned.
** The final value of *pazDequote is undefined in this case.
*/
static int fts3tokDequoteArray(
int argc, /* Number of elements in argv[] */
const char * const *argv, /* Input array */
char ***pazDequote /* Output array */
){
int rc = SQLITE_OK; /* Return code */
if( argc==0 ){
*pazDequote = 0;
}else{
int i;
int nByte = 0;
char **azDequote;
for(i=0; i<argc; i++){
nByte += (int)(strlen(argv[i]) + 1);
}
*pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
if( azDequote==0 ){
rc = SQLITE_NOMEM;
}else{
char *pSpace = (char *)&azDequote[argc];
for(i=0; i<argc; i++){
int n = (int)strlen(argv[i]);
azDequote[i] = pSpace;
memcpy(pSpace, argv[i], n+1);
sqlite3Fts3Dequote(pSpace);
pSpace += (n+1);
}
}
}
return rc;
}
/*
** Schema of the tokenizer table.
*/
#define FTS3_TOK_SCHEMA "CREATE TABLE x(input, token, start, end, position)"
/*
** This function does all the work for both the xConnect and xCreate methods.
** These tables have no persistent representation of their own, so xConnect
** and xCreate are identical operations.
**
** argv[0]: module name
** argv[1]: database name
** argv[2]: table name
** argv[3]: first argument (tokenizer name)
*/
static int fts3tokConnectMethod(
sqlite3 *db, /* Database connection */
void *pHash, /* Hash table of tokenizers */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
Fts3tokTable *pTab = 0;
const sqlite3_tokenizer_module *pMod = 0;
sqlite3_tokenizer *pTok = 0;
int rc;
char **azDequote = 0;
int nDequote;
rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nDequote = argc-3;
rc = fts3tokDequoteArray(nDequote, &argv[3], &azDequote);
if( rc==SQLITE_OK ){
const char *zModule;
if( nDequote<1 ){
zModule = "simple";
}else{
zModule = azDequote[0];
}
rc = fts3tokQueryTokenizer((Fts3Hash*)pHash, zModule, &pMod, pzErr);
}
assert( (rc==SQLITE_OK)==(pMod!=0) );
if( rc==SQLITE_OK ){
const char * const *azArg = (const char * const *)&azDequote[1];
rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok);
}
if( rc==SQLITE_OK ){
pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable));
if( pTab==0 ){
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
memset(pTab, 0, sizeof(Fts3tokTable));
pTab->pMod = pMod;
pTab->pTok = pTok;
*ppVtab = &pTab->base;
}else{
if( pTok ){
pMod->xDestroy(pTok);
}
}
sqlite3_free(azDequote);
return rc;
}
/*
** This function does the work for both the xDisconnect and xDestroy methods.
** These tables have no persistent representation of their own, so xDisconnect
** and xDestroy are identical operations.
*/
static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){
Fts3tokTable *pTab = (Fts3tokTable *)pVtab;
pTab->pMod->xDestroy(pTab->pTok);
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts3tokBestIndexMethod(
sqlite3_vtab *pVTab,
sqlite3_index_info *pInfo
){
int i;
UNUSED_PARAMETER(pVTab);
for(i=0; i<pInfo->nConstraint; i++){
if( pInfo->aConstraint[i].usable
&& pInfo->aConstraint[i].iColumn==0
&& pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ
){
pInfo->idxNum = 1;
pInfo->aConstraintUsage[i].argvIndex = 1;
pInfo->aConstraintUsage[i].omit = 1;
pInfo->estimatedCost = 1;
return SQLITE_OK;
}
}
pInfo->idxNum = 0;
assert( pInfo->estimatedCost>1000000.0 );
return SQLITE_OK;
}
/*
** xOpen - Open a cursor.
*/
static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts3tokCursor *pCsr;
UNUSED_PARAMETER(pVTab);
pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor));
if( pCsr==0 ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(Fts3tokCursor));
*ppCsr = (sqlite3_vtab_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Reset the tokenizer cursor passed as the only argument. As if it had
** just been returned by fts3tokOpenMethod().
*/
static void fts3tokResetCursor(Fts3tokCursor *pCsr){
if( pCsr->pCsr ){
Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab);
pTab->pMod->xClose(pCsr->pCsr);
pCsr->pCsr = 0;
}
sqlite3_free(pCsr->zInput);
pCsr->zInput = 0;
pCsr->zToken = 0;
pCsr->nToken = 0;
pCsr->iStart = 0;
pCsr->iEnd = 0;
pCsr->iPos = 0;
pCsr->iRowid = 0;
}
/*
** xClose - Close a cursor.
*/
static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
fts3tokResetCursor(pCsr);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** xNext - Advance the cursor to the next row, if any.
*/
static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab);
int rc; /* Return code */
pCsr->iRowid++;
rc = pTab->pMod->xNext(pCsr->pCsr,
&pCsr->zToken, &pCsr->nToken,
&pCsr->iStart, &pCsr->iEnd, &pCsr->iPos
);
if( rc!=SQLITE_OK ){
fts3tokResetCursor(pCsr);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
return rc;
}
/*
** xFilter - Initialize a cursor to point at the start of its data.
*/
static int fts3tokFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
int rc = SQLITE_ERROR;
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab);
UNUSED_PARAMETER(idxStr);
UNUSED_PARAMETER(nVal);
fts3tokResetCursor(pCsr);
if( idxNum==1 ){
const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
int nByte = sqlite3_value_bytes(apVal[0]);
pCsr->zInput = sqlite3_malloc(nByte+1);
if( pCsr->zInput==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zInput, zByte, nByte);
pCsr->zInput[nByte] = 0;
rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr);
if( rc==SQLITE_OK ){
pCsr->pCsr->pTokenizer = pTab->pTok;
}
}
}
if( rc!=SQLITE_OK ) return rc;
return fts3tokNextMethod(pCursor);
}
/*
** xEof - Return true if the cursor is at EOF, or false otherwise.
*/
static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
return (pCsr->zToken==0);
}
/*
** xColumn - Return a column value.
*/
static int fts3tokColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
/* CREATE TABLE x(input, token, start, end, position) */
switch( iCol ){
case 0:
sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
break;
case 1:
sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT);
break;
case 2:
sqlite3_result_int(pCtx, pCsr->iStart);
break;
case 3:
sqlite3_result_int(pCtx, pCsr->iEnd);
break;
default:
assert( iCol==4 );
sqlite3_result_int(pCtx, pCsr->iPos);
break;
}
return SQLITE_OK;
}
/*
** xRowid - Return the current rowid for the cursor.
*/
static int fts3tokRowidMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite_int64 *pRowid /* OUT: Rowid value */
){
Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
*pRowid = (sqlite3_int64)pCsr->iRowid;
return SQLITE_OK;
}
/*
** Register the fts3tok module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
static const sqlite3_module fts3tok_module = {
0, /* iVersion */
fts3tokConnectMethod, /* xCreate */
fts3tokConnectMethod, /* xConnect */
fts3tokBestIndexMethod, /* xBestIndex */
fts3tokDisconnectMethod, /* xDisconnect */
fts3tokDisconnectMethod, /* xDestroy */
fts3tokOpenMethod, /* xOpen */
fts3tokCloseMethod, /* xClose */
fts3tokFilterMethod, /* xFilter */
fts3tokNextMethod, /* xNext */
fts3tokEofMethod, /* xEof */
fts3tokColumnMethod, /* xColumn */
fts3tokRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0 /* xRollbackTo */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+27 -55
View File
@@ -29,18 +29,6 @@
#include <assert.h>
#include <string.h>
/*
** Return true if the two-argument version of fts3_tokenizer()
** has been activated via a prior call to sqlite3_db_config(db,
** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
*/
static int fts3TokenizerEnabled(sqlite3_context *context){
sqlite3 *db = sqlite3_context_db_handle(context);
int isEnabled = 0;
sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
return isEnabled;
}
/*
** Implementation of the SQL scalar function for accessing the underlying
** hash table. This function may be called as follows:
@@ -61,7 +49,7 @@ static int fts3TokenizerEnabled(sqlite3_context *context){
** is a blob containing the pointer stored as the hash data corresponding
** to string <key-name> (after the hash-table is updated, if applicable).
*/
static void fts3TokenizerFunc(
static void scalarFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
@@ -79,26 +67,20 @@ static void fts3TokenizerFunc(
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
if( fts3TokenizerEnabled(context) ){
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( zName==0 || n!=sizeof(pPtr) ){
sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
pPtr = *(void **)sqlite3_value_blob(argv[1]);
pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
if( pOld==pPtr ){
sqlite3_result_error(context, "out of memory", -1);
}
}else{
sqlite3_result_error(context, "fts3tokenize disabled", -1);
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( n!=sizeof(pPtr) ){
sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
pPtr = *(void **)sqlite3_value_blob(argv[1]);
pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
if( pOld==pPtr ){
sqlite3_result_error(context, "out of memory", -1);
return;
}
}else{
if( zName ){
pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
}
pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
if( !pPtr ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
@@ -106,6 +88,7 @@ static void fts3TokenizerFunc(
return;
}
}
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
@@ -178,16 +161,12 @@ int sqlite3Fts3InitTokenizer(
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
if( z==0 ){
assert( n==0 );
z = zCopy;
}
z[n] = '\0';
sqlite3Fts3Dequote(z);
m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z);
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
}else{
char const **aArg = 0;
@@ -210,7 +189,7 @@ int sqlite3Fts3InitTokenizer(
rc = m->xCreate(iArg, aArg, ppTok);
assert( rc!=SQLITE_OK || *ppTok );
if( rc!=SQLITE_OK ){
sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer");
*pzErr = sqlite3_mprintf("unknown tokenizer");
}else{
(*ppTok)->pModule = m;
}
@@ -224,11 +203,7 @@ int sqlite3Fts3InitTokenizer(
#ifdef SQLITE_TEST
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <tcl.h>
#include <string.h>
/*
@@ -298,9 +273,9 @@ static void testFunc(
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr2, -1);
sqlite3_free(zErr2);
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
@@ -369,7 +344,6 @@ int registerTokenizer(
return sqlite3_finalize(pStmt);
}
static
int queryTokenizer(
sqlite3 *db,
@@ -440,13 +414,11 @@ static void intTestFunc(
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
if( fts3TokenizerEnabled(context) ){
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
}
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
@@ -462,7 +434,7 @@ static void intTestFunc(
** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
**
** This function adds a scalar function (see header comment above
** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
** scalarFunc() in this file for details) and, if ENABLE_TABLE is
** defined at compilation time, a temporary virtual table (see header
** comment above struct HashTableVtab) to the database schema. Both
** provide read/write access to the contents of *pHash.
@@ -491,10 +463,10 @@ int sqlite3Fts3InitHashTable(
#endif
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
}
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
}
#ifdef SQLITE_TEST
if( SQLITE_OK==rc ){
+20 -20
View File
@@ -13,7 +13,7 @@
** Implementation of the "unicode" full-text-search tokenizer.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
#ifdef SQLITE_ENABLE_FTS4_UNICODE61
#include "fts3Int.h"
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
@@ -136,16 +136,16 @@ static int unicodeAddExceptions(
){
const unsigned char *z = (const unsigned char *)zIn;
const unsigned char *zTerm = &z[nIn];
unsigned int iCode;
int iCode;
int nEntry = 0;
assert( bAlnum==0 || bAlnum==1 );
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic(iCode)==0
){
nEntry++;
}
@@ -162,13 +162,13 @@ static int unicodeAddExceptions(
z = (const unsigned char *)zIn;
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic(iCode)==0
){
int i, j;
for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
for(i=0; i<nNew && aNew[i]<iCode; i++);
for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
aNew[i] = (int)iCode;
aNew[i] = iCode;
nNew++;
}
}
@@ -231,7 +231,7 @@ static int unicodeCreate(
for(i=0; rc==SQLITE_OK && i<nArg; i++){
const char *z = azArg[i];
int n = (int)strlen(z);
int n = strlen(z);
if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
pNew->bRemoveDiacritic = 1;
@@ -318,7 +318,7 @@ static int unicodeNext(
){
unicode_cursor *pCsr = (unicode_cursor *)pC;
unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
unsigned int iCode = 0;
int iCode;
char *zOut;
const unsigned char *z = &pCsr->aInput[pCsr->iOff];
const unsigned char *zStart = z;
@@ -330,7 +330,7 @@ static int unicodeNext(
** the input. */
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( unicodeIsAlnum(p, (int)iCode) ) break;
if( unicodeIsAlnum(p, iCode) ) break;
zStart = z;
}
if( zStart>=zTerm ) return SQLITE_DONE;
@@ -350,7 +350,7 @@ static int unicodeNext(
/* Write the folded case of the last character read to the output */
zEnd = z;
iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
if( iOut ){
WRITE_UTF8(zOut, iOut);
}
@@ -358,16 +358,16 @@ static int unicodeNext(
/* If the cursor is not at EOF, read the next character */
if( z>=zTerm ) break;
READ_UTF8(z, zTerm, iCode);
}while( unicodeIsAlnum(p, (int)iCode)
|| sqlite3FtsUnicodeIsdiacritic((int)iCode)
}while( unicodeIsAlnum(p, iCode)
|| sqlite3FtsUnicodeIsdiacritic(iCode)
);
/* Set the output variables and return. */
pCsr->iOff = (int)(z - pCsr->aInput);
pCsr->iOff = (z - pCsr->aInput);
*paToken = pCsr->zToken;
*pnToken = (int)(zOut - pCsr->zToken);
*piStart = (int)(zStart - pCsr->aInput);
*piEnd = (int)(zEnd - pCsr->aInput);
*pnToken = zOut - pCsr->zToken;
*piStart = (zStart - pCsr->aInput);
*piEnd = (zEnd - pCsr->aInput);
*piPos = pCsr->iToken++;
return SQLITE_OK;
}
@@ -390,4 +390,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */
+38 -36
View File
@@ -15,7 +15,7 @@
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
#if defined(SQLITE_ENABLE_FTS4_UNICODE61)
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
#include <assert.h>
@@ -39,7 +39,7 @@ int sqlite3FtsUnicodeIsalnum(int c){
** C. It is not possible to represent a range larger than 1023 codepoints
** using this format.
*/
static const unsigned int aEntry[] = {
const static unsigned int aEntry[] = {
0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
@@ -101,37 +101,38 @@ int sqlite3FtsUnicodeIsalnum(int c){
0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
0x380400F0,
0x037FFC02, 0x03E3FC01, 0x03EC7801, 0x03ECA401, 0x03EEC810,
0x03F4F802, 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023,
0x03F95013, 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807,
0x03FCEC06, 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405,
0x04040003, 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E,
0x040E7C01, 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01,
0x04280403, 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01,
0x04294009, 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016,
0x04420003, 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004,
0x04460003, 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004,
0x05BD442E, 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5,
0x07480046, 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01,
0x075C5401, 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401,
0x075EA401, 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064,
0x07C2800F, 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F,
0x07C4C03C, 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009,
0x07C94002, 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014,
0x07CE8025, 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001,
0x07D108B6, 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018,
0x07D7EC46, 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401,
0x38008060, 0x380400F0, 0x3C000001, 0x3FFFF401, 0x40000001,
0x43FFF401,
};
static const unsigned int aAscii[4] = {
0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
};
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
if( c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iRes;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
@@ -202,7 +203,7 @@ static int remove_diacritic(int c){
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
};
/*
@@ -322,17 +323,16 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
int ret = c;
assert( c>=0 );
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
@@ -343,12 +343,14 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
iHi = iTest-1;
}
}
assert( iRes<0 || c>=aEntry[iRes].iCode );
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
if( iRes>=0 ){
const struct TableEntry *p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
}
if( bRemoveDiacritic ) ret = remove_diacritic(ret);
@@ -361,4 +363,4 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */
#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */
+151 -423
View File
@@ -193,7 +193,6 @@ struct SegmentWriter {
int nSize; /* Size of allocation at aData */
int nData; /* Bytes of data in aData */
char *aData; /* Pointer to block from malloc() */
i64 nLeafData; /* Number of bytes of leaf data written */
};
/*
@@ -269,10 +268,6 @@ struct SegmentNode {
#define SQL_SELECT_INDEXES 35
#define SQL_SELECT_MXLEVEL 36
#define SQL_SELECT_LEVEL_RANGE2 37
#define SQL_UPDATE_LEVEL_IDX 38
#define SQL_UPDATE_LEVEL 39
/*
** This function is used to obtain an SQLite prepared statement handle
** for the statement identified by the second argument. If successful,
@@ -326,15 +321,14 @@ static int fts3SqlStmt(
/* 25 */ "",
/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows. */
/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
" GROUP BY level HAVING cnt>=?"
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
" ORDER BY (level %% 1024) ASC LIMIT 1",
/* Estimate the upper limit on the number of leaf nodes in a new segment
@@ -375,18 +369,7 @@ static int fts3SqlStmt(
/* SQL_SELECT_MXLEVEL
** Return the largest relative level in the FTS index or indexes. */
/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'",
/* Return segments in order from oldest to newest.*/
/* 37 */ "SELECT level, idx, end_block "
"FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? "
"ORDER BY level DESC, idx ASC",
/* Update statements used while promoting segments */
/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? "
"WHERE level=? AND idx=?",
/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1"
/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'"
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@@ -407,8 +390,7 @@ static int fts3SqlStmt(
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
&pStmt, NULL);
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
sqlite3_free(zSql);
assert( rc==SQLITE_OK || pStmt==0 );
p->aStmt[eStmt] = pStmt;
@@ -507,30 +489,37 @@ static void fts3SqlExec(
/*
** This function ensures that the caller has obtained an exclusive
** shared-cache table-lock on the %_segdir table. This is required before
** writing data to the fts3 table. If this lock is not acquired first, then
** the caller may end up attempting to take this lock as part of committing
** a transaction, causing SQLite to return SQLITE_LOCKED or
** LOCKED_SHAREDCACHEto a COMMIT command.
** This function ensures that the caller has obtained a shared-cache
** table-lock on the %_content table. This is required before reading
** data from the fts3 table. If this lock is not acquired first, then
** the caller may end up holding read-locks on the %_segments and %_segdir
** tables, but no read-lock on the %_content table. If this happens
** a second connection will be able to write to the fts3 table, but
** attempting to commit those writes might return SQLITE_LOCKED or
** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain
** write-locks on the %_segments and %_segdir ** tables).
**
** It is best to avoid this because if FTS3 returns any error when
** committing a transaction, the whole transaction will be rolled back.
** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE.
** It can still happen if the user locks the underlying tables directly
** instead of accessing them via FTS.
** We try to avoid this because if FTS3 returns any error when committing
** a transaction, the whole transaction will be rolled back. And this is
** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can
** still happen if the user reads data directly from the %_segments or
** %_segdir tables instead of going through FTS3 though.
**
** This reasoning does not apply to a content=xxx table.
*/
static int fts3Writelock(Fts3Table *p){
int rc = SQLITE_OK;
if( p->nPendingData==0 ){
sqlite3_stmt *pStmt;
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0);
int sqlite3Fts3ReadLock(Fts3Table *p){
int rc; /* Return code */
sqlite3_stmt *pStmt; /* Statement used to obtain lock */
if( p->zContentTbl==0 ){
rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_null(pStmt, 1);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
}
}else{
rc = SQLITE_OK;
}
return rc;
@@ -862,12 +851,10 @@ static int fts3PendingTermsAdd(
*/
static int fts3PendingTermsDocid(
Fts3Table *p, /* Full-text table handle */
int bDelete, /* True if this op is a delete */
int iLangid, /* Language id of row being written */
sqlite_int64 iDocid /* Docid of row being written */
){
assert( iLangid>=0 );
assert( bDelete==1 || bDelete==0 );
/* TODO(shess) Explore whether partially flushing the buffer on
** forced-flush would provide better performance. I suspect that if
@@ -875,8 +862,7 @@ static int fts3PendingTermsDocid(
** buffer was half empty, that would let the less frequent terms
** generate longer doclists.
*/
if( iDocid<p->iPrevDocid
|| (iDocid==p->iPrevDocid && p->bPrevDelete==0)
if( iDocid<=p->iPrevDocid
|| p->iPrevLangid!=iLangid
|| p->nPendingData>p->nMaxPendingData
){
@@ -885,7 +871,6 @@ static int fts3PendingTermsDocid(
}
p->iPrevDocid = iDocid;
p->iPrevLangid = iLangid;
p->bPrevDelete = bDelete;
return SQLITE_OK;
}
@@ -922,15 +907,12 @@ static int fts3InsertTerms(
){
int i; /* Iterator variable */
for(i=2; i<p->nColumn+2; i++){
int iCol = i-2;
if( p->abNotindexed[iCol]==0 ){
const char *zText = (const char *)sqlite3_value_text(apVal[i]);
int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]);
if( rc!=SQLITE_OK ){
return rc;
}
aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
const char *zText = (const char *)sqlite3_value_text(apVal[i]);
int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]);
if( rc!=SQLITE_OK ){
return rc;
}
aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
}
return SQLITE_OK;
}
@@ -1075,15 +1057,11 @@ static void fts3DeleteTerms(
if( SQLITE_ROW==sqlite3_step(pSelect) ){
int i;
int iLangid = langidFromSelect(p, pSelect);
i64 iDocid = sqlite3_column_int64(pSelect, 0);
rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid);
rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
int iCol = i-1;
if( p->abNotindexed[iCol]==0 ){
const char *zText = (const char *)sqlite3_column_text(pSelect, i);
rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
const char *zText = (const char *)sqlite3_column_text(pSelect, i);
rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
if( rc!=SQLITE_OK ){
sqlite3_reset(pSelect);
@@ -1324,19 +1302,14 @@ static int fts3SegReaderNext(
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
sqlite3_free(pReader->aNode);
pReader->aNode = 0;
if( pElem ){
char *aCopy;
if( pElem==0 ){
pReader->aNode = 0;
}else{
PendingList *pList = (PendingList *)fts3HashData(pElem);
int nCopy = pList->nData+1;
pReader->zTerm = (char *)fts3HashKey(pElem);
pReader->nTerm = fts3HashKeysize(pElem);
aCopy = (char*)sqlite3_malloc(nCopy);
if( !aCopy ) return SQLITE_NOMEM;
memcpy(aCopy, pList->aData, nCopy);
pReader->nNode = pReader->nDoclist = nCopy;
pReader->aNode = pReader->aDoclist = aCopy;
pReader->nNode = pReader->nDoclist = pList->nData + 1;
pReader->aNode = pReader->aDoclist = pList->aData;
pReader->ppNextElem++;
assert( pReader->aNode );
}
@@ -1372,8 +1345,8 @@ static int fts3SegReaderNext(
/* Because of the FTS3_NODE_PADDING bytes of padding, the following is
** safe (no risk of overread) even if the node data is corrupted. */
pNext += fts3GetVarint32(pNext, &nPrefix);
pNext += fts3GetVarint32(pNext, &nSuffix);
pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
if( nPrefix<0 || nSuffix<=0
|| &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
){
@@ -1396,7 +1369,7 @@ static int fts3SegReaderNext(
memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
pReader->nTerm = nPrefix+nSuffix;
pNext += nSuffix;
pNext += fts3GetVarint32(pNext, &pReader->nDoclist);
pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
pReader->aDoclist = pNext;
pReader->pOffsetList = 0;
@@ -1489,7 +1462,7 @@ static int fts3SegReaderNextDocid(
/* The following line of code (and the "p++" below the while() loop) is
** normally all that is required to move pointer p to the desired
** position. The exception is if this node is being loaded from disk
** incrementally and pointer "p" now points to the first byte past
** incrementally and pointer "p" now points to the first byte passed
** the populated part of pReader->aNode[].
*/
while( *p | c ) c = *p++ & 0x80;
@@ -1576,14 +1549,12 @@ int sqlite3Fts3MsrOvfl(
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
if( pReader ){
if( !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
}
if( pReader && !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
if( !fts3SegReaderIsRootOnly(pReader) ){
sqlite3_free(pReader->aNode);
sqlite3_blob_close(pReader->pBlob);
}
sqlite3_blob_close(pReader->pBlob);
}
sqlite3_free(pReader);
}
@@ -1639,10 +1610,7 @@ int sqlite3Fts3SegReaderNew(
** an array of pending terms by term. This occurs as part of flushing
** the contents of the pending-terms hash table to the database.
*/
static int SQLITE_CDECL fts3CompareElemByTerm(
const void *lhs,
const void *rhs
){
static int fts3CompareElemByTerm(const void *lhs, const void *rhs){
char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
@@ -1943,7 +1911,6 @@ static int fts3WriteSegdir(
sqlite3_int64 iStartBlock, /* Value for "start_block" field */
sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
sqlite3_int64 iEndBlock, /* Value for "end_block" field */
sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */
char *zRoot, /* Blob value for "root" field */
int nRoot /* Number of bytes in buffer zRoot */
){
@@ -1954,13 +1921,7 @@ static int fts3WriteSegdir(
sqlite3_bind_int(pStmt, 2, iIdx);
sqlite3_bind_int64(pStmt, 3, iStartBlock);
sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
if( nLeafData==0 ){
sqlite3_bind_int64(pStmt, 5, iEndBlock);
}else{
char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData);
if( !zEnd ) return SQLITE_NOMEM;
sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
}
sqlite3_bind_int64(pStmt, 5, iEndBlock);
sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
@@ -2286,9 +2247,6 @@ static int fts3SegWriterAdd(
nDoclist; /* Doclist data */
}
/* Increase the total number of bytes written to account for the new entry. */
pWriter->nLeafData += nReq;
/* If the buffer currently allocated is too small for this entry, realloc
** the buffer to make it large enough.
*/
@@ -2360,13 +2318,13 @@ static int fts3SegWriterFlush(
pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
}
if( rc==SQLITE_OK ){
rc = fts3WriteSegdir(p, iLevel, iIdx,
pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot);
rc = fts3WriteSegdir(
p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
}
}else{
/* The entire tree fits on the root node. Write it to the segdir table. */
rc = fts3WriteSegdir(p, iLevel, iIdx,
0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData);
rc = fts3WriteSegdir(
p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
}
p->nLeafAdd++;
return rc;
@@ -2450,37 +2408,6 @@ static int fts3SegmentMaxLevel(
return sqlite3_reset(pStmt);
}
/*
** iAbsLevel is an absolute level that may be assumed to exist within
** the database. This function checks if it is the largest level number
** within its index. Assuming no error occurs, *pbMax is set to 1 if
** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK
** is returned. If an error occurs, an error code is returned and the
** final value of *pbMax is undefined.
*/
static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){
/* Set pStmt to the compiled version of:
**
** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
**
** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
*/
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pStmt, 1, iAbsLevel+1);
sqlite3_bind_int64(pStmt, 2,
((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
);
*pbMax = 0;
if( SQLITE_ROW==sqlite3_step(pStmt) ){
*pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL;
}
return sqlite3_reset(pStmt);
}
/*
** Delete all entries in the %_segments table associated with the segment
** opened with seg-reader pSeg. This function does not affect the contents
@@ -2603,7 +2530,7 @@ static void fts3ColumnFilter(
break;
}
p = &pList[1];
p += fts3GetVarint32(p, &iCurrent);
p += sqlite3Fts3GetVarint32(p, &iCurrent);
}
if( bZero && &pList[nList]!=pEnd ){
@@ -2922,8 +2849,8 @@ int sqlite3Fts3SegReaderStep(
fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
while( apSegment[0]->pOffsetList ){
int j; /* Number of segments that share a docid */
char *pList = 0;
int nList = 0;
char *pList;
int nList;
int nByte;
sqlite3_int64 iDocid = apSegment[0]->iDocid;
fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
@@ -3016,140 +2943,6 @@ void sqlite3Fts3SegReaderFinish(
}
}
/*
** Decode the "end_block" field, selected by column iCol of the SELECT
** statement passed as the first argument.
**
** The "end_block" field may contain either an integer, or a text field
** containing the text representation of two non-negative integers separated
** by one or more space (0x20) characters. In the first case, set *piEndBlock
** to the integer value and *pnByte to zero before returning. In the second,
** set *piEndBlock to the first value and *pnByte to the second.
*/
static void fts3ReadEndBlockField(
sqlite3_stmt *pStmt,
int iCol,
i64 *piEndBlock,
i64 *pnByte
){
const unsigned char *zText = sqlite3_column_text(pStmt, iCol);
if( zText ){
int i;
int iMul = 1;
i64 iVal = 0;
for(i=0; zText[i]>='0' && zText[i]<='9'; i++){
iVal = iVal*10 + (zText[i] - '0');
}
*piEndBlock = iVal;
while( zText[i]==' ' ) i++;
iVal = 0;
if( zText[i]=='-' ){
i++;
iMul = -1;
}
for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){
iVal = iVal*10 + (zText[i] - '0');
}
*pnByte = (iVal * (i64)iMul);
}
}
/*
** A segment of size nByte bytes has just been written to absolute level
** iAbsLevel. Promote any segments that should be promoted as a result.
*/
static int fts3PromoteSegments(
Fts3Table *p, /* FTS table handle */
sqlite3_int64 iAbsLevel, /* Absolute level just updated */
sqlite3_int64 nByte /* Size of new segment at iAbsLevel */
){
int rc = SQLITE_OK;
sqlite3_stmt *pRange;
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0);
if( rc==SQLITE_OK ){
int bOk = 0;
i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1;
i64 nLimit = (nByte*3)/2;
/* Loop through all entries in the %_segdir table corresponding to
** segments in this index on levels greater than iAbsLevel. If there is
** at least one such segment, and it is possible to determine that all
** such segments are smaller than nLimit bytes in size, they will be
** promoted to level iAbsLevel. */
sqlite3_bind_int64(pRange, 1, iAbsLevel+1);
sqlite3_bind_int64(pRange, 2, iLast);
while( SQLITE_ROW==sqlite3_step(pRange) ){
i64 nSize = 0, dummy;
fts3ReadEndBlockField(pRange, 2, &dummy, &nSize);
if( nSize<=0 || nSize>nLimit ){
/* If nSize==0, then the %_segdir.end_block field does not not
** contain a size value. This happens if it was written by an
** old version of FTS. In this case it is not possible to determine
** the size of the segment, and so segment promotion does not
** take place. */
bOk = 0;
break;
}
bOk = 1;
}
rc = sqlite3_reset(pRange);
if( bOk ){
int iIdx = 0;
sqlite3_stmt *pUpdate1 = 0;
sqlite3_stmt *pUpdate2 = 0;
if( rc==SQLITE_OK ){
rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0);
}
if( rc==SQLITE_OK ){
rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0);
}
if( rc==SQLITE_OK ){
/* Loop through all %_segdir entries for segments in this index with
** levels equal to or greater than iAbsLevel. As each entry is visited,
** updated it to set (level = -1) and (idx = N), where N is 0 for the
** oldest segment in the range, 1 for the next oldest, and so on.
**
** In other words, move all segments being promoted to level -1,
** setting the "idx" fields as appropriate to keep them in the same
** order. The contents of level -1 (which is never used, except
** transiently here), will be moved back to level iAbsLevel below. */
sqlite3_bind_int64(pRange, 1, iAbsLevel);
while( SQLITE_ROW==sqlite3_step(pRange) ){
sqlite3_bind_int(pUpdate1, 1, iIdx++);
sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0));
sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1));
sqlite3_step(pUpdate1);
rc = sqlite3_reset(pUpdate1);
if( rc!=SQLITE_OK ){
sqlite3_reset(pRange);
break;
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3_reset(pRange);
}
/* Move level -1 to level iAbsLevel */
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pUpdate2, 1, iAbsLevel);
sqlite3_step(pUpdate2);
rc = sqlite3_reset(pUpdate2);
}
}
}
return rc;
}
/*
** Merge all level iLevel segments in the database into a single
** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
@@ -3174,7 +2967,6 @@ static int fts3SegmentMerge(
Fts3SegFilter filter; /* Segment term filter condition */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */
i64 iMaxLevel = 0; /* Max level number for this index/langid */
assert( iLevel==FTS3_SEGCURSOR_ALL
|| iLevel==FTS3_SEGCURSOR_PENDING
@@ -3186,35 +2978,30 @@ static int fts3SegmentMerge(
rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr);
if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel);
if( rc!=SQLITE_OK ) goto finished;
}
if( iLevel==FTS3_SEGCURSOR_ALL ){
/* This call is to merge all segments in the database to a single
** segment. The level of the new segment is equal to the numerically
** greatest segment level currently present in the database for this
** index. The idx of the new segment is always 0. */
if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){
if( csr.nSegment==1 ){
rc = SQLITE_DONE;
goto finished;
}
iNewLevel = iMaxLevel;
rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
bIgnoreEmpty = 1;
}else if( iLevel==FTS3_SEGCURSOR_PENDING ){
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0);
rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx);
}else{
/* This call is to merge all segments at level iLevel. find the next
** available segment index at level iLevel+1. The call to
** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
** a single iLevel+2 segment if necessary. */
assert( FTS3_SEGCURSOR_PENDING==-1 );
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel);
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
}
if( rc!=SQLITE_OK ) goto finished;
assert( csr.nSegment>0 );
assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
@@ -3231,7 +3018,7 @@ static int fts3SegmentMerge(
csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
}
if( rc!=SQLITE_OK ) goto finished;
assert( pWriter || bIgnoreEmpty );
assert( pWriter );
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3DeleteSegdir(
@@ -3239,14 +3026,7 @@ static int fts3SegmentMerge(
);
if( rc!=SQLITE_OK ) goto finished;
}
if( pWriter ){
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
if( rc==SQLITE_OK ){
if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevel<iMaxLevel ){
rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData);
}
}
}
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
finished:
fts3SegWriterFree(pWriter);
@@ -3256,7 +3036,7 @@ static int fts3SegmentMerge(
/*
** Flush the contents of pendingTerms to level 0 segments.
** Flush the contents of pendingTerms to level 0 segments.
*/
int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
int rc = SQLITE_OK;
@@ -3272,19 +3052,14 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
** estimate the number of leaf blocks of content to be written
*/
if( rc==SQLITE_OK && p->bHasStat
&& p->nAutoincrmerge==0xff && p->nLeafAdd>0
&& p->bAutoincrmerge==0xff && p->nLeafAdd>0
){
sqlite3_stmt *pStmt = 0;
rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
rc = sqlite3_step(pStmt);
if( rc==SQLITE_ROW ){
p->nAutoincrmerge = sqlite3_column_int(pStmt, 0);
if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8;
}else if( rc==SQLITE_DONE ){
p->nAutoincrmerge = 0;
}
p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
rc = sqlite3_reset(pStmt);
}
}
@@ -3458,8 +3233,7 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
int rc2;
sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
sqlite3_bind_int(pAllLangid, 2, p->nIndex);
sqlite3_bind_int(pAllLangid, 1, p->nIndex);
while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
int i;
int iLangid = sqlite3_column_int(pAllLangid, 0);
@@ -3526,14 +3300,12 @@ static int fts3DoRebuild(Fts3Table *p){
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int iCol;
int iLangid = langidFromSelect(p, pStmt);
rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0));
rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1));
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
if( p->abNotindexed[iCol]==0 ){
const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
}
const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
}
if( p->bHasDocsize ){
fts3InsertDocsize(&rc, p, aSz);
@@ -3653,8 +3425,6 @@ struct IncrmergeWriter {
int iIdx; /* Index of *output* segment in iAbsLevel+1 */
sqlite3_int64 iStart; /* Block number of first allocated block */
sqlite3_int64 iEnd; /* Block number of last allocated block */
sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */
u8 bNoLeafData; /* If true, store 0 for segment size */
NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};
@@ -3723,9 +3493,9 @@ static int nodeReaderNext(NodeReader *p){
p->aNode = 0;
}else{
if( bFirst==0 ){
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
}
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
if( rc==SQLITE_OK ){
@@ -3733,7 +3503,7 @@ static int nodeReaderNext(NodeReader *p){
p->term.n = nPrefix+nSuffix;
p->iOff += nSuffix;
if( p->iChild==0 ){
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
p->aDoclist = &p->aNode[p->iOff];
p->iOff += p->nDoclist;
}
@@ -3993,8 +3763,8 @@ static int fts3IncrmergeAppend(
nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
}
pWriter->nLeafData += nSpace;
blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc);
if( rc==SQLITE_OK ){
if( pLeaf->block.n==0 ){
pLeaf->block.n = 1;
@@ -4093,7 +3863,6 @@ static void fts3IncrmergeRelease(
pWriter->iStart, /* start_block */
pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */
pWriter->iEnd, /* end_block */
(pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */
pRoot->block.a, pRoot->block.n /* root */
);
}
@@ -4195,11 +3964,7 @@ static int fts3IncrmergeLoad(
if( sqlite3_step(pSelect)==SQLITE_ROW ){
iStart = sqlite3_column_int64(pSelect, 1);
iLeafEnd = sqlite3_column_int64(pSelect, 2);
fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
if( pWriter->nLeafData<0 ){
pWriter->nLeafData = pWriter->nLeafData * -1;
}
pWriter->bNoLeafData = (pWriter->nLeafData==0);
iEnd = sqlite3_column_int64(pSelect, 3);
nRoot = sqlite3_column_bytes(pSelect, 4);
aRoot = sqlite3_column_blob(pSelect, 4);
}else{
@@ -4790,8 +4555,8 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
pHint->n = i;
i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
i += fts3GetVarint32(&pHint->a[i], pnInput);
if( i!=nHint ) return FTS_CORRUPT_VTAB;
i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput);
if( i!=nHint ) return SQLITE_CORRUPT_VTAB;
return SQLITE_OK;
}
@@ -4800,11 +4565,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
** Incremental merges happen nMin segments at a time. The segments
** to be merged are the nMin oldest segments (the ones with the smallest
** values for the _segdir.idx field) in the highest level that contains
** at least nMin segments. Multiple merges might occur in an attempt to
** write the quota of nMerge leaf blocks.
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the
** quota of nMerge leaf blocks.
*/
int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
int rc; /* Return code */
@@ -4829,7 +4594,6 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
int bUseHint = 0; /* True if attempting to append */
int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
/* Search the %_segdir table for the absolute level with the smallest
** relative level number that contains at least nMin segments, if any.
@@ -4838,11 +4602,10 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
** set nSeg to -1.
*/
rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin));
sqlite3_bind_int(pFindLevel, 1, nMin);
if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
nSeg = sqlite3_column_int(pFindLevel, 1);
assert( nSeg>=2 );
nSeg = nMin;
}else{
nSeg = -1;
}
@@ -4884,19 +4647,6 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
** to start work on some other level. */
memset(pWriter, 0, nAlloc);
pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
if( rc==SQLITE_OK ){
rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
assert( bUseHint==1 || bUseHint==0 );
if( iIdx==0 || (bUseHint && iIdx==1) ){
int bIgnore = 0;
rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore);
if( bIgnore ){
pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
}
}
}
if( rc==SQLITE_OK ){
rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
}
@@ -4904,12 +4654,16 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
&& SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
){
if( bUseHint && iIdx>0 ){
const char *zKey = pCsr->zTerm;
int nKey = pCsr->nTerm;
rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
}else{
rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
if( rc==SQLITE_OK ){
if( bUseHint && iIdx>0 ){
const char *zKey = pCsr->zTerm;
int nKey = pCsr->nTerm;
rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
}else{
rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
}
}
if( rc==SQLITE_OK && pWriter->nLeafEst ){
@@ -4931,13 +4685,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
}
}
if( nSeg!=0 ){
pWriter->nLeafData = pWriter->nLeafData * -1;
}
fts3IncrmergeRelease(p, pWriter, &rc);
if( nSeg==0 && pWriter->bNoLeafData==0 ){
fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
}
}
sqlite3Fts3SegReaderFinish(pCsr);
@@ -4957,14 +4705,11 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
** Convert the text beginning at *pz into an integer and return
** its value. Advance *pz to point to the first character past
** the integer.
**
** This function used for parameters to merge= and incrmerge=
** commands.
*/
static int fts3Getint(const char **pz){
const char *z = *pz;
int i = 0;
while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
*pz = z;
return i;
}
@@ -5027,19 +4772,16 @@ static int fts3DoAutoincrmerge(
){
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
p->nAutoincrmerge = fts3Getint(&zParam);
if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
p->nAutoincrmerge = 8;
}
p->bAutoincrmerge = fts3Getint(&zParam)!=0;
if( !p->bHasStat ){
assert( p->bFts4==0 );
sqlite3Fts3CreateStatTable(&rc, p);
if( rc ) return rc;
}
rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
if( rc ) return rc;
if( rc ) return rc;;
sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge);
sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
return rc;
@@ -5163,8 +4905,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
int rc2;
sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
sqlite3_bind_int(pAllLangid, 2, p->nIndex);
sqlite3_bind_int(pAllLangid, 1, p->nIndex);
while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
int iLangid = sqlite3_column_int(pAllLangid, 0);
int i;
@@ -5177,6 +4918,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
}
/* This block calculates the checksum according to the %_content table */
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
sqlite3_stmt *pStmt = 0;
@@ -5196,36 +4938,34 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int iCol;
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
if( p->abNotindexed[iCol]==0 ){
const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
int nText = sqlite3_column_bytes(pStmt, iCol+1);
sqlite3_tokenizer_cursor *pT = 0;
const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
int nText = sqlite3_column_bytes(pStmt, iCol+1);
sqlite3_tokenizer_cursor *pT = 0;
rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken = 0; /* Number of bytes in token */
int iDum1 = 0, iDum2 = 0; /* Dummy variables */
int iPos = 0; /* Position of token in zText */
rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken = 0; /* Number of bytes in token */
int iDum1 = 0, iDum2 = 0; /* Dummy variables */
int iPos = 0; /* Position of token in zText */
rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
if( rc==SQLITE_OK ){
int i;
cksum2 = cksum2 ^ fts3ChecksumEntry(
zToken, nToken, iLang, 0, iDocid, iCol, iPos
);
for(i=1; i<p->nIndex; i++){
if( p->aIndex[i].nPrefix<=nToken ){
cksum2 = cksum2 ^ fts3ChecksumEntry(
zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
);
}
rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
if( rc==SQLITE_OK ){
int i;
cksum2 = cksum2 ^ fts3ChecksumEntry(
zToken, nToken, iLang, 0, iDocid, iCol, iPos
);
for(i=1; i<p->nIndex; i++){
if( p->aIndex[i].nPrefix<=nToken ){
cksum2 = cksum2 ^ fts3ChecksumEntry(
zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
);
}
}
}
if( pT ) pModule->xClose(pT);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
if( pT ) pModule->xClose(pT);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
}
@@ -5273,7 +5013,7 @@ static int fts3DoIntegrityCheck(
int rc;
int bOk = 0;
rc = fts3IntegrityCheck(p, &bOk);
if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
return rc;
}
@@ -5309,9 +5049,6 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
}else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
p->nMaxPendingData = atoi(&zVal[11]);
rc = SQLITE_OK;
}else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){
p->bNoIncrDoclist = atoi(&zVal[21]);
rc = SQLITE_OK;
#endif
}else{
rc = SQLITE_ERROR;
@@ -5371,34 +5108,32 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
if( p->abNotindexed[i]==0 ){
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken = 0; /* Number of bytes in token */
int iDum1 = 0, iDum2 = 0; /* Dummy variables */
int iPos = 0; /* Position of token in zText */
rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
&& (pPT->bFirst==0 || iPos==0)
&& (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
&& (0==memcmp(zToken, pPT->z, pPT->n))
){
fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
}
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken = 0; /* Number of bytes in token */
int iDum1 = 0, iDum2 = 0; /* Dummy variables */
int iPos = 0; /* Position of token in zText */
rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
&& (pPT->bFirst==0 || iPos==0)
&& (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
&& (0==memcmp(zToken, pPT->z, pPT->n))
){
fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
}
}
if( pTC ) pModule->xClose(pTC);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
if( pTC ) pModule->xClose(pTC);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
if( pDef->pList ){
rc = fts3PendingListAppendVarint(&pDef->pList, 0);
@@ -5530,10 +5265,6 @@ int sqlite3Fts3UpdateMethod(
int nChng = 0; /* Net change in number of documents */
int bInsertDone = 0;
/* At this point it must be known if the %_stat table exists or not.
** So bHasStat may not be 2. */
assert( p->bHasStat==0 || p->bHasStat==1 );
assert( p->pSegments==0 );
assert(
nArg==1 /* DELETE operations */
@@ -5566,9 +5297,6 @@ int sqlite3Fts3UpdateMethod(
aSzIns = &aSzDel[p->nColumn+1];
memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2);
rc = fts3Writelock(p);
if( rc!=SQLITE_OK ) goto update_out;
/* If this is an INSERT operation, or an UPDATE that modifies the rowid
** value, then this operation requires constraint handling.
**
@@ -5635,7 +5363,7 @@ int sqlite3Fts3UpdateMethod(
}
}
if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
assert( p->iPrevDocid==*pRowid );
-16
View File
@@ -1,16 +0,0 @@
#!/bin/sh
set -e
srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
echo ""
for f in `ls $srcdir/ext/fts3/*.c`
do
f=`basename $f`
echo -ne "$f: "
gcov -b $f | grep Taken | sed 's/Taken at least once://'
done
+10 -11
View File
@@ -376,7 +376,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
sqlite3_finalize(pStmt);
nLeaf = nSeg - nIdx;
printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n",
pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0);
pgsz-45, n, n*100.0/nLeaf);
pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab);
mxLevel = 0;
@@ -398,8 +398,8 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
if( sqlite3_step(pStmt)==SQLITE_ROW
&& (nLeaf = sqlite3_column_int(pStmt, 0))>0
){
int nIdx = sqlite3_column_int(pStmt, 5);
sqlite3_int64 sz;
nIdx = sqlite3_column_int(pStmt, 5);
printf("For level %d:\n", i);
printf(" Number of indexes...................... %9d\n", nIdx);
printf(" Number of leaf segments................ %9d\n", nLeaf);
@@ -504,7 +504,7 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){
sqlite3_column_int64(pStmt,5));
printf(" root %9s\n", rtag);
if( iLEnd>iStart ){
sqlite3_int64 iLower, iPrev = 0, iX;
sqlite3_int64 iLower, iPrev, iX;
if( iLEnd+1<=iEnd ){
sqlite3_bind_int64(pStmt2, 1, iLEnd+1);
sqlite3_bind_int64(pStmt2, 2, iEnd);
@@ -548,13 +548,13 @@ static void decodeSegment(
const unsigned char *aData, /* Content to print */
int nData /* Number of bytes of content */
){
sqlite3_int64 iChild = 0;
sqlite3_int64 iChild;
sqlite3_int64 iPrefix;
sqlite3_int64 nTerm;
sqlite3_int64 n;
sqlite3_int64 iDocsz;
int iHeight;
sqlite3_int64 i = 0;
int i = 0;
int cnt = 0;
char zTerm[1000];
@@ -576,12 +576,12 @@ static void decodeSegment(
fprintf(stderr, "term to long\n");
exit(1);
}
memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm);
memcpy(zTerm+iPrefix, aData+i, nTerm);
zTerm[iPrefix+nTerm] = 0;
i += nTerm;
if( iHeight==0 ){
i += getVarint(aData+i, &iDocsz);
printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i);
printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i);
i += iDocsz;
}else{
printf("term: %-25s child %lld\n", zTerm, ++iChild);
@@ -749,19 +749,18 @@ static void decodeDoclist(
*/
static void showDoclist(sqlite3 *db, const char *zTab){
const unsigned char *aData;
sqlite3_int64 offset;
int nData;
sqlite3_int64 offset, nData;
sqlite3_stmt *pStmt;
offset = atoi64(azExtra[1]);
nData = atoi(azExtra[2]);
nData = atoi64(azExtra[2]);
pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
if( sqlite3_step(pStmt)!=SQLITE_ROW ){
sqlite3_finalize(pStmt);
return;
}
aData = sqlite3_column_blob(pStmt, 0);
printf("Doclist at %s offset %lld of size %d bytes:\n",
printf("Doclist at %s offset %lld of size %lld bytes:\n",
azExtra[0], offset, nData);
if( findOption("raw", 0, 0)!=0 ){
printBlob(aData+offset, nData);
+172 -58
View File
@@ -1,5 +1,77 @@
source [file join [file dirname [info script]] parseunicode.tcl]
#
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the
# codepoint should simply be removed from the input. Examples:
#
# { 224 a } (replace codepoint 224 to "a")
# { 769 "" } (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
proc rd_load_unicodedata_text {zName} {
global tl_lookup_table
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
if { [llength $character_decomposition_mapping]!=2
|| [string is xdigit [lindex $character_decomposition_mapping 0]]==0
} {
continue
}
set iCode [expr "0x$code"]
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
if {[info exists tl_lookup_table($iCode)]} continue
if { ($iAscii >= 97 && $iAscii <= 122)
|| ($iAscii >= 65 && $iAscii <= 90)
} {
lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
set dia($iDia) 1
}
}
foreach d [array names dia] {
lappend lRet [list $d ""]
}
set lRet [lsort -integer -index 0 $lRet]
close $fd
set lRet
}
proc print_rd {map} {
global tl_lookup_table
@@ -45,7 +117,7 @@ proc print_rd {map} {
puts "** E\"). The resuls of passing a codepoint that corresponds to an"
puts "** uppercase letter are undefined."
puts "*/"
puts "static int ${::remove_diacritic}(int c)\{"
puts "static int remove_diacritic(int c)\{"
puts " unsigned short aDia\[\] = \{"
puts -nonewline " 0, "
set i 1
@@ -88,7 +160,7 @@ proc print_rd {map} {
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
puts "\}"
puts "\};"
}
proc print_isdiacritic {zFunc map} {
@@ -132,6 +204,50 @@ proc print_isdiacritic {zFunc map} {
#-------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of codepoints (integers). The list
# contains all codepoints in the UnicodeData.txt assigned to any "General
# Category" that is not a "Letter" or "Number".
#
proc an_load_unicodedata_text {zName} {
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
set iCode [expr "0x$code"]
set bAlnum [expr {[lsearch {L N} [string range $general_category 0 0]]>=0}]
if { !$bAlnum } { lappend lRet $iCode }
}
close $fd
set lRet
}
proc an_load_separator_ranges {} {
global unicodedata.txt
set lSep [an_load_unicodedata_text ${unicodedata.txt}]
@@ -179,7 +295,7 @@ proc an_print_range_array {lRange} {
** using this format.
*/
}]
puts -nonewline " static const unsigned int aEntry\[\] = \{"
puts -nonewline " const static unsigned int aEntry\[\] = \{"
set i 0
foreach range $lRange {
foreach {iFirst nRange} $range {}
@@ -226,11 +342,11 @@ proc print_isalnum {zFunc lRange} {
an_print_range_array $lRange
an_print_ascii_bitmap $lRange
puts {
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
if( c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iRes;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
@@ -244,7 +360,7 @@ proc print_isalnum {zFunc lRange} {
}
assert( aEntry[0]<key );
assert( key>=aEntry[iRes] );
return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
return (c >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
}
return 1;}
puts "\}"
@@ -321,6 +437,29 @@ proc print_test_isalnum {zFunc lRange} {
#-------------------------------------------------------------------------
proc tl_load_casefolding_txt {zName} {
global tl_lookup_table
set fd [open $zName]
while { ![eof $fd] } {
set line [gets $fd]
if {[string range $line 0 0] == "#"} continue
if {$line == ""} continue
foreach x {a b c d} {unset -nocomplain $x}
foreach {a b c d} [split $line ";"] {}
set a2 [list]
set c2 [list]
foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
set b [string trim $b]
set d [string trim $d]
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
}
}
proc tl_create_records {} {
global tl_lookup_table
@@ -484,20 +623,19 @@ proc print_fold {zFunc} {
tl_print_table_footer toggle
tl_print_ioff_table $liOff
puts [subst -nocommands {
puts {
int ret = c;
assert( c>=0 );
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
@@ -508,17 +646,19 @@ proc print_fold {zFunc} {
iHi = iTest-1;
}
}
assert( iRes<0 || c>=aEntry[iRes].iCode );
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
if( iRes>=0 ){
const struct TableEntry *p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
}
if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
if( bRemoveDiacritic ) ret = remove_diacritic(ret);
}
}
}]
foreach entry $lHigh {
tl_print_if_entry $entry
@@ -589,12 +729,8 @@ proc print_fileheader {} {
*/
}]
puts ""
if {$::generate_fts5_code} {
# no-op
} else {
puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
}
puts "#if !defined(SQLITE_DISABLE_FTS3_UNICODE)"
puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
puts ""
puts "#include <assert.h>"
puts ""
@@ -621,40 +757,22 @@ proc print_test_main {} {
# our liking.
#
proc usage {} {
puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? "
puts -nonewline stderr "Usage: $::argv0 ?-test? "
puts stderr "<CaseFolding.txt file> <UnicodeData.txt file>"
exit 1
}
if {[llength $argv]<2} usage
if {[llength $argv]!=2 && [llength $argv]!=3} usage
if {[llength $argv]==3 && [lindex $argv 0]!="-test"} usage
set unicodedata.txt [lindex $argv end]
set casefolding.txt [lindex $argv end-1]
set remove_diacritic remove_diacritic
set generate_test_code 0
set generate_fts5_code 0
set function_prefix "sqlite3Fts"
for {set i 0} {$i < [llength $argv]-2} {incr i} {
switch -- [lindex $argv $i] {
-test {
set generate_test_code 1
}
-fts5 {
set function_prefix sqlite3Fts5
set generate_fts5_code 1
set remove_diacritic fts5_remove_diacritic
}
default {
usage
}
}
}
set generate_test_code [expr {[llength $argv]==3}]
print_fileheader
# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
print_isalnum ${function_prefix}UnicodeIsalnum $lRange
print_isalnum sqlite3FtsUnicodeIsalnum $lRange
# Leave a gap between the two generated C functions.
#
@@ -669,26 +787,22 @@ set mappings [rd_load_unicodedata_text ${unicodedata.txt}]
print_rd $mappings
puts ""
puts ""
print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
print_isdiacritic sqlite3FtsUnicodeIsdiacritic $mappings
puts ""
puts ""
# Print the fold() function to stdout.
#
print_fold ${function_prefix}UnicodeFold
print_fold sqlite3FtsUnicodeFold
# Print the test routines and main() function to stdout, if -test
# was specified.
#
if {$::generate_test_code} {
print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
print_fold_test ${function_prefix}UnicodeFold $mappings
print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange
print_fold_test sqlite3FtsUnicodeFold $mappings
print_test_main
}
if {$generate_fts5_code} {
# no-op
} else {
puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}
puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
-146
View File
@@ -1,146 +0,0 @@
#--------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the
# codepoint should simply be removed from the input. Examples:
#
# { 224 a } (replace codepoint 224 to "a")
# { 769 "" } (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
proc rd_load_unicodedata_text {zName} {
global tl_lookup_table
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
if { [llength $character_decomposition_mapping]!=2
|| [string is xdigit [lindex $character_decomposition_mapping 0]]==0
} {
continue
}
set iCode [expr "0x$code"]
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
if {[info exists tl_lookup_table($iCode)]} continue
if { ($iAscii >= 97 && $iAscii <= 122)
|| ($iAscii >= 65 && $iAscii <= 90)
} {
lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
set dia($iDia) 1
}
}
foreach d [array names dia] {
lappend lRet [list $d ""]
}
set lRet [lsort -integer -index 0 $lRet]
close $fd
set lRet
}
#-------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of codepoints (integers). The list
# contains all codepoints in the UnicodeData.txt assigned to any "General
# Category" that is not a "Letter" or "Number".
#
proc an_load_unicodedata_text {zName} {
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
set iCode [expr "0x$code"]
set bAlnum [expr {
[lsearch {L N} [string range $general_category 0 0]] >= 0
|| $general_category=="Co"
}]
if { !$bAlnum } { lappend lRet $iCode }
}
close $fd
set lRet
}
proc tl_load_casefolding_txt {zName} {
global tl_lookup_table
set fd [open $zName]
while { ![eof $fd] } {
set line [gets $fd]
if {[string range $line 0 0] == "#"} continue
if {$line == ""} continue
foreach x {a b c d} {unset -nocomplain $x}
foreach {a b c d} [split $line ";"] {}
set a2 [list]
set c2 [list]
foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
set b [string trim $b]
set d [string trim $d]
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
}
}
-252
View File
@@ -1,252 +0,0 @@
#
# 2014 August 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#--------------------------------------------------------------------------
#
# This script extracts the documentation for the API used by fts5 auxiliary
# functions from header file fts5.h. It outputs html text on stdout that
# is included in the documentation on the web.
#
set ::fts5_docs_output ""
if {[info commands hd_putsnl]==""} {
if {[llength $argv]>0} { set ::extract_api_docs_mode [lindex $argv 0] }
proc output {text} {
puts $text
}
} else {
proc output {text} {
append ::fts5_docs_output "$text\n"
}
}
if {[info exists ::extract_api_docs_mode]==0} {set ::extract_api_docs_mode api}
set input_file [file join [file dir [info script]] fts5.h]
set fd [open $input_file]
set data [read $fd]
close $fd
# Argument $data is the entire text of the fts5.h file. This function
# extracts the definition of the Fts5ExtensionApi structure from it and
# returns a key/value list of structure member names and definitions. i.e.
#
# iVersion {int iVersion} xUserData {void *(*xUserData)(Fts5Context*)} ...
#
proc get_struct_members {data} {
# Extract the structure definition from the fts5.h file.
regexp "struct Fts5ExtensionApi {(.*?)};" $data -> defn
# Remove all comments from the structure definition
regsub -all {/[*].*?[*]/} $defn {} defn2
set res [list]
foreach member [split $defn2 {;}] {
set member [string trim $member]
if {$member!=""} {
catch { set name [lindex $member end] }
regexp {.*?[(][*]([^)]*)[)]} $member -> name
lappend res $name $member
}
}
set res
}
proc get_struct_docs {data names} {
# Extract the structure definition from the fts5.h file.
regexp {EXTENSION API FUNCTIONS(.*?)[*]/} $data -> docs
set current_doc ""
set current_header ""
foreach line [split $docs "\n"] {
regsub {[*]*} $line {} line
if {[regexp {^ } $line]} {
append current_doc "$line\n"
} elseif {[string trim $line]==""} {
if {$current_header!=""} { append current_doc "\n" }
} else {
if {$current_doc != ""} {
lappend res $current_header $current_doc
set current_doc ""
}
set subject n/a
regexp {^ *([[:alpha:]]*)} $line -> subject
if {[lsearch $names $subject]>=0} {
set current_header $subject
} else {
set current_header [string trim $line]
}
}
}
if {$current_doc != ""} {
lappend res $current_header $current_doc
}
set res
}
proc get_tokenizer_docs {data} {
regexp {(xCreate:.*?)[*]/} $data -> docs
set res "<dl>\n"
foreach line [split [string trim $docs] "\n"] {
regexp {[*][*](.*)} $line -> line
if {[regexp {^ ?x.*:} $line]} {
append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
continue
}
if {[regexp {SYNONYM SUPPORT} $line]} {
set line "</dl><h3>Synonym Support</h3>"
}
if {[string trim $line] == ""} {
append res "<p>\n"
} else {
append res "$line\n"
}
}
set res
}
proc get_api_docs {data} {
# Initialize global array M as a map from Fts5StructureApi member name
# to member definition. i.e.
#
# iVersion -> {int iVersion}
# xUserData -> {void *(*xUserData)(Fts5Context*)}
# ...
#
array set M [get_struct_members $data]
# Initialize global list D as a map from section name to documentation
# text. Most (all?) section names are structure member names.
#
set D [get_struct_docs $data [array names M]]
output "<dl>"
foreach {sub docs} $D {
if {[info exists M($sub)]} {
set hdr $M($sub)
set link " id=$sub"
} else {
set link ""
}
#output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
#set style "padding-left:6ex;font-size:1.4em;display:block"
#output "<h style=\"$style\"><pre>$hdr</pre></h>"
regsub -line {^ *[)]} $hdr ")" hdr
output "<dt style=\"white-space:pre;font-family:monospace;font-size:120%\""
output "$link>"
output "<b>$hdr</b></dt><dd>"
set mode ""
set margin " style=margin-top:0.1em"
foreach line [split [string trim $docs] "\n"] {
if {[string trim $line]==""} {
if {$mode != ""} {output "</$mode>"}
set mode ""
} elseif {$mode == ""} {
if {[regexp {^ } $line]} {
set mode codeblock
} else {
set mode p
}
output "<$mode$margin>"
set margin ""
}
output $line
}
if {$mode != ""} {output "</$mode>"}
output "</dd>"
}
output "</dl>"
}
proc get_fts5_struct {data start end} {
set res ""
set bOut 0
foreach line [split $data "\n"] {
if {$bOut==0} {
if {[regexp $start $line]} {
set bOut 1
}
}
if {$bOut} {
append res "$line\n"
}
if {$bOut} {
if {[regexp $end $line]} {
set bOut 0
}
}
}
set map [list /* <i>/* */ */</i>]
string map $map $res
}
proc main {data} {
switch $::extract_api_docs_mode {
fts5_api {
output [get_fts5_struct $data "typedef struct fts5_api" "^\};"]
}
fts5_tokenizer {
output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"]
output [get_fts5_struct $data \
"Flags that may be passed as the third argument to xTokenize()" \
"#define FTS5_TOKEN_COLOCATED"
]
}
fts5_extension {
output [get_fts5_struct $data "typedef.*Fts5ExtensionApi" "^.;"]
}
Fts5ExtensionApi {
set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
set map [list]
foreach {k v} [get_struct_members $data] {
if {[string match x* $k]==0} continue
lappend map $k "<a href=#$k>$k</a>"
}
output [string map $map $struct]
}
api {
get_api_docs $data
}
tokenizer_api {
output [get_tokenizer_docs $data]
}
default {
}
}
}
main $data
set ::fts5_docs_output
-579
View File
@@ -1,579 +0,0 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Interfaces to extend FTS5. Using the interfaces defined in this file,
** FTS5 may be extended with:
**
** * custom tokenizers, and
** * custom auxiliary functions.
*/
#ifndef _FTS5_H
#define _FTS5_H
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif
/*************************************************************************
** CUSTOM AUXILIARY FUNCTIONS
**
** Virtual table implementations may overload SQL functions by implementing
** the sqlite3_module.xFindFunction() method.
*/
typedef struct Fts5ExtensionApi Fts5ExtensionApi;
typedef struct Fts5Context Fts5Context;
typedef struct Fts5PhraseIter Fts5PhraseIter;
typedef void (*fts5_extension_function)(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
);
struct Fts5PhraseIter {
const unsigned char *a;
const unsigned char *b;
};
/*
** EXTENSION API FUNCTIONS
**
** xUserData(pFts):
** Return a copy of the context pointer the extension function was
** registered with.
**
** xColumnTotalSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
** to the total number of tokens in the FTS5 table. Or, if iCol is
** non-negative but less than the number of columns in the table, return
** the total number of tokens in column iCol, considering all rows in
** the FTS5 table.
**
** If parameter iCol is greater than or equal to the number of columns
** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
** an OOM condition or IO error), an appropriate SQLite error code is
** returned.
**
** xColumnCount(pFts):
** Return the number of columns in the table.
**
** xColumnSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
** to the total number of tokens in the current row. Or, if iCol is
** non-negative but less than the number of columns in the table, set
** *pnToken to the number of tokens in column iCol of the current row.
**
** If parameter iCol is greater than or equal to the number of columns
** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
** an OOM condition or IO error), an appropriate SQLite error code is
** returned.
**
** This function may be quite inefficient if used with an FTS5 table
** created with the "columnsize=0" option.
**
** xColumnText:
** This function attempts to retrieve the text of column iCol of the
** current document. If successful, (*pz) is set to point to a buffer
** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
** if an error occurs, an SQLite error code is returned and the final values
** of (*pz) and (*pn) are undefined.
**
** xPhraseCount:
** Returns the number of phrases in the current query expression.
**
** xPhraseSize:
** Returns the number of tokens in phrase iPhrase of the query. Phrases
** are numbered starting from zero.
**
** xInstCount:
** Set *pnInst to the total number of occurrences of all phrases within
** the query within the current row. Return SQLITE_OK if successful, or
** an error code (i.e. SQLITE_NOMEM) if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option. If the FTS5 table is created
** with either "detail=none" or "detail=column" and "content=" option
** (i.e. if it is a contentless table), then this API always returns 0.
**
** xInst:
** Query for the details of phrase match iIdx within the current row.
** Phrase matches are numbered starting from zero, so the iIdx argument
** should be greater than or equal to zero and smaller than the value
** output by xInstCount().
**
** Usually, output parameter *piPhrase is set to the phrase number, *piCol
** to the column in which it occurs and *piOff the token offset of the
** first token of the phrase. The exception is if the table was created
** with the offsets=0 option specified. In this case *piOff is always
** set to -1.
**
** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM)
** if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
**
** xRowid:
** Returns the rowid of the current row.
**
** xTokenize:
** Tokenize text using the tokenizer belonging to the FTS5 table.
**
** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
** This API function is used to query the FTS table for phrase iPhrase
** of the current query. Specifically, a query equivalent to:
**
** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
**
** with $p set to a phrase equivalent to the phrase iPhrase of the
** current query is executed. Any column filter that applies to
** phrase iPhrase of the current query is included in $p. For each
** row visited, the callback function passed as the fourth argument
** is invoked. The context and API objects passed to the callback
** function may be used to access the properties of each matched row.
** Invoking Api.xUserData() returns a copy of the pointer passed as
** the third argument to pUserData.
**
** If the callback function returns any value other than SQLITE_OK, the
** query is abandoned and the xQueryPhrase function returns immediately.
** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
** Otherwise, the error code is propagated upwards.
**
** If the query runs to completion without incident, SQLITE_OK is returned.
** Or, if some error occurs before the query completes or is aborted by
** the callback, an SQLite error code is returned.
**
**
** xSetAuxdata(pFts5, pAux, xDelete)
**
** Save the pointer passed as the second argument as the extension functions
** "auxiliary data". The pointer may then be retrieved by the current or any
** future invocation of the same fts5 extension function made as part of
** of the same MATCH query using the xGetAuxdata() API.
**
** Each extension function is allocated a single auxiliary data slot for
** each FTS query (MATCH expression). If the extension function is invoked
** more than once for a single FTS query, then all invocations share a
** single auxiliary data context.
**
** If there is already an auxiliary data pointer when this function is
** invoked, then it is replaced by the new pointer. If an xDelete callback
** was specified along with the original pointer, it is invoked at this
** point.
**
** The xDelete callback, if one is specified, is also invoked on the
** auxiliary data pointer after the FTS5 query has finished.
**
** If an error (e.g. an OOM condition) occurs within this function, an
** the auxiliary data is set to NULL and an error code returned. If the
** xDelete parameter was not NULL, it is invoked on the auxiliary data
** pointer before returning.
**
**
** xGetAuxdata(pFts5, bClear)
**
** Returns the current auxiliary data pointer for the fts5 extension
** function. See the xSetAuxdata() method for details.
**
** If the bClear argument is non-zero, then the auxiliary data is cleared
** (set to NULL) before this function returns. In this case the xDelete,
** if any, is not invoked.
**
**
** xRowCount(pFts5, pnRow)
**
** This function is used to retrieve the total number of rows in the table.
** In other words, the same value that would be returned by:
**
** SELECT count(*) FROM ftstable;
**
** xPhraseFirst()
** This function is used, along with type Fts5PhraseIter and the xPhraseNext
** method, to iterate through all instances of a single query phrase within
** the current row. This is the same information as is accessible via the
** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
** to use, this API may be faster under some circumstances. To iterate
** through instances of phrase iPhrase, use the following code:
**
** Fts5PhraseIter iter;
** int iCol, iOff;
** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
** iCol>=0;
** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
** ){
** // An instance of phrase iPhrase at offset iOff of column iCol
** }
**
** The Fts5PhraseIter structure is defined above. Applications should not
** modify this structure directly - it should only be used as shown above
** with the xPhraseFirst() and xPhraseNext() API methods (and by
** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option. If the FTS5 table is created
** with either "detail=none" or "detail=column" and "content=" option
** (i.e. if it is a contentless table), then this API always iterates
** through an empty set (all calls to xPhraseFirst() set iCol to -1).
**
** xPhraseNext()
** See xPhraseFirst above.
**
** xPhraseFirstColumn()
** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
** and xPhraseNext() APIs described above. The difference is that instead
** of iterating through all instances of a phrase in the current row, these
** APIs are used to iterate through the set of columns in the current row
** that contain one or more instances of a specified phrase. For example:
**
** Fts5PhraseIter iter;
** int iCol;
** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
** iCol>=0;
** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
** ){
** // Column iCol contains at least one instance of phrase iPhrase
** }
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" option. If the FTS5 table is created with either
** "detail=none" "content=" option (i.e. if it is a contentless table),
** then this API always iterates through an empty set (all calls to
** xPhraseFirstColumn() set iCol to -1).
**
** The information accessed using this API and its companion
** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
** (or xInst/xInstCount). The chief advantage of this API is that it is
** significantly more efficient than those alternatives when used with
** "detail=column" tables.
**
** xPhraseNextColumn()
** See xPhraseFirstColumn above.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 3 */
void *(*xUserData)(Fts5Context*);
int (*xColumnCount)(Fts5Context*);
int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
int (*xTokenize)(Fts5Context*,
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
int (*xPhraseCount)(Fts5Context*);
int (*xPhraseSize)(Fts5Context*, int iPhrase);
int (*xInstCount)(Fts5Context*, int *pnInst);
int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
sqlite3_int64 (*xRowid)(Fts5Context*);
int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
);
int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
void *(*xGetAuxdata)(Fts5Context*, int bClear);
int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
};
/*
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
/*************************************************************************
** CUSTOM TOKENIZERS
**
** Applications may also register custom tokenizer types. A tokenizer
** is registered by providing fts5 with a populated instance of the
** following structure. All structure methods must be defined, setting
** any member of the fts5_tokenizer struct to NULL leads to undefined
** behaviour. The structure methods are expected to function as follows:
**
** xCreate:
** This function is used to allocate and initialize a tokenizer instance.
** A tokenizer instance is required to actually tokenize text.
**
** The first argument passed to this function is a copy of the (void*)
** pointer provided by the application when the fts5_tokenizer object
** was registered with FTS5 (the third argument to xCreateTokenizer()).
** The second and third arguments are an array of nul-terminated strings
** containing the tokenizer arguments, if any, specified following the
** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
** to create the FTS5 table.
**
** The final argument is an output variable. If successful, (*ppOut)
** should be set to point to the new tokenizer handle and SQLITE_OK
** returned. If an error occurs, some value other than SQLITE_OK should
** be returned. In this case, fts5 assumes that the final value of *ppOut
** is undefined.
**
** xDelete:
** This function is invoked to delete a tokenizer handle previously
** allocated using xCreate(). Fts5 guarantees that this function will
** be invoked exactly once for each successful call to xCreate().
**
** xTokenize:
** This function is expected to tokenize the nText byte string indicated
** by argument pText. pText may or may not be nul-terminated. The first
** argument passed to this function is a pointer to an Fts5Tokenizer object
** returned by an earlier call to xCreate().
**
** The second argument indicates the reason that FTS5 is requesting
** tokenization of the supplied text. This is always one of the following
** four values:
**
** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
** or removed from the FTS table. The tokenizer is being invoked to
** determine the set of tokens to add to (or delete from) the
** FTS index.
**
** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
** against the FTS index. The tokenizer is being called to tokenize
** a bareword or quoted string specified as part of the query.
**
** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
** followed by a "*" character, indicating that the last token
** returned by the tokenizer will be treated as a token prefix.
**
** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
** satisfy an fts5_api.xTokenize() request made by an auxiliary
** function. Or an fts5_api.xColumnSize() request made by the same
** on a columnsize=0 database.
** </ul>
**
** For each token in the input string, the supplied callback xToken() must
** be invoked. The first argument to it should be a copy of the pointer
** passed as the second argument to xTokenize(). The third and fourth
** arguments are a pointer to a buffer containing the token text, and the
** size of the token in bytes. The 4th and 5th arguments are the byte offsets
** of the first byte of and first byte immediately following the text from
** which the token is derived within the input.
**
** The second argument passed to the xToken() callback ("tflags") should
** normally be set to 0. The exception is if the tokenizer supports
** synonyms. In this case see the discussion below for details.
**
** FTS5 assumes the xToken() callback is invoked for each token in the
** order that they occur within the input text.
**
** If an xToken() callback returns any value other than SQLITE_OK, then
** the tokenization should be abandoned and the xTokenize() method should
** immediately return a copy of the xToken() return value. Or, if the
** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
** if an error occurs with the xTokenize() implementation itself, it
** may abandon the tokenization and return any error code other than
** SQLITE_OK or SQLITE_DONE.
**
** SYNONYM SUPPORT
**
** Custom tokenizers may also support synonyms. Consider a case in which a
** user wishes to query for a phrase such as "first place". Using the
** built-in tokenizers, the FTS5 query 'first + place' will match instances
** of "first place" within the document set, but not alternative forms
** such as "1st place". In some applications, it would be better to match
** all instances of "first place" or "1st place" regardless of which form
** the user specified in the MATCH query text.
**
** There are several ways to approach this in FTS5:
**
** <ol><li> By mapping all synonyms to a single token. In this case, the
** In the above example, this means that the tokenizer returns the
** same token for inputs "first" and "1st". Say that token is in
** fact "first", so that when the user inserts the document "I won
** 1st place" entries are added to the index for tokens "i", "won",
** "first" and "place". If the user then queries for '1st + place',
** the tokenizer substitutes "first" for "1st" and the query works
** as expected.
**
** <li> By adding multiple synonyms for a single term to the FTS index.
** In this case, when tokenizing query text, the tokenizer may
** provide multiple synonyms for a single term within the document.
** FTS5 then queries the index for each synonym individually. For
** example, faced with the query:
**
** <codeblock>
** ... MATCH 'first place'</codeblock>
**
** the tokenizer offers both "1st" and "first" as synonyms for the
** first token in the MATCH query and FTS5 effectively runs a query
** similar to:
**
** <codeblock>
** ... MATCH '(first OR 1st) place'</codeblock>
**
** except that, for the purposes of auxiliary functions, the query
** still appears to contain just two phrases - "(first OR 1st)"
** being treated as a single phrase.
**
** <li> By adding multiple synonyms for a single term to the FTS index.
** Using this method, when tokenizing document text, the tokenizer
** provides multiple synonyms for each token. So that when a
** document such as "I won first place" is tokenized, entries are
** added to the FTS index for "i", "won", "first", "1st" and
** "place".
**
** This way, even if the tokenizer does not provide synonyms
** when tokenizing query text (it should not - to do would be
** inefficient), it doesn't matter if the user queries for
** 'first + place' or '1st + place', as there are entires in the
** FTS index corresponding to both forms of the first token.
** </ol>
**
** Whether it is parsing document or query text, any call to xToken that
** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
** is considered to supply a synonym for the previous token. For example,
** when parsing the document "I won first place", a tokenizer that supports
** synonyms would call xToken() 5 times, as follows:
**
** <codeblock>
** xToken(pCtx, 0, "i", 1, 0, 1);
** xToken(pCtx, 0, "won", 3, 2, 5);
** xToken(pCtx, 0, "first", 5, 6, 11);
** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
** xToken(pCtx, 0, "place", 5, 12, 17);
**</codeblock>
**
** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
** xToken() is called. Multiple synonyms may be specified for a single token
** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
** There is no limit to the number of synonyms that may be provided for a
** single token.
**
** In many cases, method (1) above is the best approach. It does not add
** extra data to the FTS index or require FTS5 to query for multiple terms,
** so it is efficient in terms of disk space and query speed. However, it
** does not support prefix queries very well. If, as suggested above, the
** token "first" is subsituted for "1st" by the tokenizer, then the query:
**
** <codeblock>
** ... MATCH '1s*'</codeblock>
**
** will not match documents that contain the token "1st" (as the tokenizer
** will probably not map "1s" to any prefix of "first").
**
** For full prefix support, method (3) may be preferred. In this case,
** because the index contains entries for both "first" and "1st", prefix
** queries such as 'fi*' or '1s*' will match correctly. However, because
** extra entries are added to the FTS index, this method uses more space
** within the database.
**
** Method (2) offers a midpoint between (1) and (3). Using this method,
** a query such as '1s*' will match documents that contain the literal
** token "1st", but not "first" (assuming the tokenizer is not able to
** provide synonyms for prefixes). However, a non-prefix query like '1st'
** will match against "1st" and "first". This method does not require
** extra disk space, as no extra entries are added to the FTS index.
** On the other hand, it may require more CPU cycles to run MATCH queries,
** as separate queries of the FTS index are required for each synonym.
**
** When using methods (2) or (3), it is important that the tokenizer only
** provide synonyms when tokenizing document text (method (2)) or query
** text (method (3)), not both. Doing so will not cause any errors, but is
** inefficient.
*/
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct fts5_tokenizer fts5_tokenizer;
struct fts5_tokenizer {
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
void (*xDelete)(Fts5Tokenizer*);
int (*xTokenize)(Fts5Tokenizer*,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
int (*xToken)(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
)
);
};
/* Flags that may be passed as the third argument to xTokenize() */
#define FTS5_TOKENIZE_QUERY 0x0001
#define FTS5_TOKENIZE_PREFIX 0x0002
#define FTS5_TOKENIZE_DOCUMENT 0x0004
#define FTS5_TOKENIZE_AUX 0x0008
/* Flags that may be passed by the tokenizer implementation back to FTS5
** as the third argument to the supplied xToken callback. */
#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
/*
** END OF CUSTOM TOKENIZERS
*************************************************************************/
/*************************************************************************
** FTS5 EXTENSION REGISTRATION API
*/
typedef struct fts5_api fts5_api;
struct fts5_api {
int iVersion; /* Currently always set to 2 */
/* Create a new tokenizer */
int (*xCreateTokenizer)(
fts5_api *pApi,
const char *zName,
void *pContext,
fts5_tokenizer *pTokenizer,
void (*xDestroy)(void*)
);
/* Find an existing tokenizer */
int (*xFindTokenizer)(
fts5_api *pApi,
const char *zName,
void **ppContext,
fts5_tokenizer *pTokenizer
);
/* Create a new auxiliary function */
int (*xCreateFunction)(
fts5_api *pApi,
const char *zName,
void *pContext,
fts5_extension_function xFunction,
void (*xDestroy)(void*)
);
};
/*
** END OF REGISTRATION API
*************************************************************************/
#ifdef __cplusplus
} /* end of the 'extern "C"' block */
#endif
#endif /* _FTS5_H */
-794
View File
@@ -1,794 +0,0 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#ifndef _FTS5INT_H
#define _FTS5INT_H
#include "fts5.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <assert.h>
#ifndef SQLITE_AMALGAMATION
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned short u16;
typedef short i16;
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
#ifndef ArraySize
# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
#endif
#define testcase(x)
#define ALWAYS(x) 1
#define NEVER(x) 0
#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#define MAX(x,y) (((x) > (y)) ? (x) : (y))
/*
** Constants for the largest and smallest possible 64-bit signed integers.
*/
# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
#endif
/* Truncate very long tokens to this many bytes. Hard limit is
** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
** field that occurs at the start of each leaf page (see fts5_index.c). */
#define FTS5_MAX_TOKEN_SIZE 32768
/*
** Maximum number of prefix indexes on single FTS5 table. This must be
** less than 32. If it is set to anything large than that, an #error
** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31
#define FTS5_DEFAULT_NEARDIST 10
#define FTS5_DEFAULT_RANK "bm25"
/* Name of rank and rowid columns */
#define FTS5_RANK_NAME "rank"
#define FTS5_ROWID_NAME "rowid"
#ifdef SQLITE_DEBUG
# define FTS5_CORRUPT sqlite3Fts5Corrupt()
int sqlite3Fts5Corrupt(void);
#else
# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
#endif
/*
** The assert_nc() macro is similar to the assert() macro, except that it
** is used for assert() conditions that are true only if it can be
** guranteed that the database is not corrupt.
*/
#ifdef SQLITE_DEBUG
extern int sqlite3_fts5_may_be_corrupt;
# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
#else
# define assert_nc(x) assert(x)
#endif
/* Mark a function parameter as unused, to suppress nuisance compiler
** warnings. */
#ifndef UNUSED_PARAM
# define UNUSED_PARAM(X) (void)(X)
#endif
#ifndef UNUSED_PARAM2
# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y)
#endif
typedef struct Fts5Global Fts5Global;
typedef struct Fts5Colset Fts5Colset;
/* If a NEAR() clump or phrase may only match a specific set of columns,
** then an object of the following type is used to record the set of columns.
** Each entry in the aiCol[] array is a column that may be matched.
**
** This object is used by fts5_expr.c and fts5_index.c.
*/
struct Fts5Colset {
int nCol;
int aiCol[1];
};
/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/
typedef struct Fts5Config Fts5Config;
/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
**
** And all information loaded from the %_config table.
**
** nAutomerge:
** The minimum number of segments that an auto-merge operation should
** attempt to merge together. A value of 1 sets the object to use the
** compile time default. Zero disables auto-merge altogether.
**
** zContent:
**
** zContentRowid:
** The value of the content_rowid= option, if one was specified. Or
** the string "rowid" otherwise. This text is not quoted - if it is
** used as part of an SQL statement it needs to be quoted appropriately.
**
** zContentExprlist:
**
** pzErrmsg:
** This exists in order to allow the fts5_index.c module to return a
** decent error message if it encounters a file-format version it does
** not understand.
**
** bColumnsize:
** True if the %_docsize table is created.
**
** bPrefixIndex:
** This is only used for debugging. If set to false, any prefix indexes
** are ignored. This value is configured using:
**
** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
**
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
char *zDb; /* Database holding FTS index (e.g. "main") */
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
char **azCol; /* Column names */
u8 *abUnindexed; /* True for unindexed columns */
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
int eContent; /* An FTS5_CONTENT value */
char *zContent; /* content table */
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
/* Values loaded from the %_config table */
int iCookie; /* Incremented when %_config is modified */
int pgsz; /* Approximate page size used in %_data */
int nAutomerge; /* 'automerge' setting */
int nCrisisMerge; /* Maximum allowed segments per level */
int nUsermerge; /* 'usermerge' setting */
int nHashSize; /* Bytes of memory for in-memory hash */
char *zRank; /* Name of rank function */
char *zRankArgs; /* Arguments to rank function */
/* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
char **pzErrmsg;
#ifdef SQLITE_DEBUG
int bPrefixIndex; /* True to use prefix-indexes */
#endif
};
/* Current expected value of %_config table 'version' field */
#define FTS5_CURRENT_VERSION 4
#define FTS5_CONTENT_NORMAL 0
#define FTS5_CONTENT_NONE 1
#define FTS5_CONTENT_EXTERNAL 2
#define FTS5_DETAIL_FULL 0
#define FTS5_DETAIL_NONE 1
#define FTS5_DETAIL_COLUMNS 2
int sqlite3Fts5ConfigParse(
Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
);
void sqlite3Fts5ConfigFree(Fts5Config*);
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
void sqlite3Fts5Dequote(char *z);
/* Load the contents of the %_config table */
int sqlite3Fts5ConfigLoad(Fts5Config*, int);
/* Set the value of a single config attribute */
int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_buffer.c.
*/
/*
** Buffer object for the incremental building of string data.
*/
typedef struct Fts5Buffer Fts5Buffer;
struct Fts5Buffer {
u8 *p;
int n;
int nSpace;
};
int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
void sqlite3Fts5BufferFree(Fts5Buffer*);
void sqlite3Fts5BufferZero(Fts5Buffer*);
void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
#define fts5BufferGrow(pRc,pBuf,nn) ( \
(u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
)
/* Write and decode big-endian 32-bit integer values */
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
typedef struct Fts5PoslistReader Fts5PoslistReader;
struct Fts5PoslistReader {
/* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
const u8 *a; /* Position list to iterate through */
int n; /* Size of buffer at a[] in bytes */
int i; /* Current offset in a[] */
u8 bFlag; /* For client use (any custom purpose) */
/* Output variables */
u8 bEof; /* Set to true at EOF */
i64 iPos; /* (iCol<<32) + iPos */
};
int sqlite3Fts5PoslistReaderInit(
const u8 *a, int n, /* Poslist buffer to iterate through */
Fts5PoslistReader *pIter /* Iterator object to initialize */
);
int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
typedef struct Fts5PoslistWriter Fts5PoslistWriter;
struct Fts5PoslistWriter {
i64 iPrev;
};
int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
int sqlite3Fts5PoslistNext64(
const u8 *a, int n, /* Buffer containing poslist */
int *pi, /* IN/OUT: Offset within a[] */
i64 *piOff /* IN/OUT: Current offset */
);
/* Malloc utility */
void *sqlite3Fts5MallocZero(int *pRc, int nByte);
char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
/* Character set tests (like isspace(), isalpha() etc.) */
int sqlite3Fts5IsBareword(char t);
/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
typedef struct Fts5Termset Fts5Termset;
int sqlite3Fts5TermsetNew(Fts5Termset**);
int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
void sqlite3Fts5TermsetFree(Fts5Termset*);
/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_index.c. fts5_index.c contains contains code
** to access the data stored in the %_data table.
*/
typedef struct Fts5Index Fts5Index;
typedef struct Fts5IndexIter Fts5IndexIter;
struct Fts5IndexIter {
i64 iRowid;
const u8 *pData;
int nData;
u8 bEof;
};
#define sqlite3Fts5IterEof(x) ((x)->bEof)
/*
** Values used as part of the flags argument passed to IndexQuery().
*/
#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
/* The following are used internally by the fts5_index.c module. They are
** defined here only to make it easier to avoid clashes with the flags
** above. */
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
/*
** Create/destroy an Fts5Index object.
*/
int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
int sqlite3Fts5IndexClose(Fts5Index *p);
/*
** Return a simple checksum value based on the arguments.
*/
u64 sqlite3Fts5IndexEntryCksum(
i64 iRowid,
int iCol,
int iPos,
int iIdx,
const char *pTerm,
int nTerm
);
/*
** Argument p points to a buffer containing utf-8 text that is n bytes in
** size. Return the number of bytes in the nChar character prefix of the
** buffer, or 0 if there are less than nChar characters in total.
*/
int sqlite3Fts5IndexCharlenToBytelen(
const char *p,
int nByte,
int nChar
);
/*
** Open a new iterator to iterate though all rowids that match the
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
Fts5Index *p, /* FTS index to query */
const char *pToken, int nToken, /* Token (or prefix) to query for */
int flags, /* Mask of FTS5INDEX_QUERY_X flags */
Fts5Colset *pColset, /* Match these columns only */
Fts5IndexIter **ppIter /* OUT: New iterator object */
);
/*
** The various operations on open token or token prefix iterators opened
** using sqlite3Fts5IndexQuery().
*/
int sqlite3Fts5IterNext(Fts5IndexIter*);
int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
/*
** Close an iterator opened by sqlite3Fts5IndexQuery().
*/
void sqlite3Fts5IterClose(Fts5IndexIter*);
/*
** This interface is used by the fts5vocab module.
*/
const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
int sqlite3Fts5IterNextScan(Fts5IndexIter*);
/*
** Insert or remove data to or from the index. Each time a document is
** added to or removed from the index, this function is called one or more
** times.
**
** For an insert, it must be called once for each token in the new document.
** If the operation is a delete, it must be called (at least) once for each
** unique token in the document with an iCol value less than zero. The iPos
** argument is ignored for a delete.
*/
int sqlite3Fts5IndexWrite(
Fts5Index *p, /* Index to write to */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
const char *pToken, int nToken /* Token to add or remove to or from index */
);
/*
** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
** document iDocid.
*/
int sqlite3Fts5IndexBeginWrite(
Fts5Index *p, /* Index to write to */
int bDelete, /* True if current operation is a delete */
i64 iDocid /* Docid to add or remove data from */
);
/*
** Flush any data stored in the in-memory hash tables to the database.
** Also close any open blob handles.
*/
int sqlite3Fts5IndexSync(Fts5Index *p);
/*
** Discard any data stored in the in-memory hash tables. Do not write it
** to the database. Additionally, assume that the contents of the %_data
** table may have changed on disk. So any in-memory caches of %_data
** records must be invalidated.
*/
int sqlite3Fts5IndexRollback(Fts5Index *p);
/*
** Get or set the "averages" values.
*/
int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
/*
** Functions called by the storage module as part of integrity-check.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
/*
** Called during virtual module initialization to register UDF
** fts5_decode() with SQLite
*/
int sqlite3Fts5IndexInit(sqlite3*);
int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
/*
** Return the total number of entries read from the %_data table by
** this connection since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p);
int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexReset(Fts5Index *p);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_varint.c.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint sqlite3Fts5GetVarint
#define fts5FastGetVarint32(a, iOff, nVal) { \
nVal = (a)[iOff++]; \
if( nVal & 0x80 ){ \
iOff--; \
iOff += fts5GetVarint32(&(a)[iOff], nVal); \
} \
}
/*
** End of interface to code in fts5_varint.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5.c.
*/
int sqlite3Fts5GetTokenizer(
Fts5Global*,
const char **azArg,
int nArg,
Fts5Tokenizer**,
fts5_tokenizer**,
char **pzErr
);
Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
/*
** End of interface to code in fts5.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_hash.c.
*/
typedef struct Fts5Hash Fts5Hash;
/*
** Create a hash table, free a hash table.
*/
int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
void sqlite3Fts5HashFree(Fts5Hash*);
int sqlite3Fts5HashWrite(
Fts5Hash*,
i64 iRowid, /* Rowid for this entry */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
char bByte,
const char *pToken, int nToken /* Token to add or remove to or from index */
);
/*
** Empty (but do not delete) a hash table.
*/
void sqlite3Fts5HashClear(Fts5Hash*);
int sqlite3Fts5HashQuery(
Fts5Hash*, /* Hash table to query */
const char *pTerm, int nTerm, /* Query term */
const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
int *pnDoclist /* OUT: Size of doclist in bytes */
);
int sqlite3Fts5HashScanInit(
Fts5Hash*, /* Hash table to query */
const char *pTerm, int nTerm /* Query prefix */
);
void sqlite3Fts5HashScanNext(Fts5Hash*);
int sqlite3Fts5HashScanEof(Fts5Hash*);
void sqlite3Fts5HashScanEntry(Fts5Hash *,
const char **pzTerm, /* OUT: term (nul-terminated) */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
);
/*
** End of interface to code in fts5_hash.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains
** code to access the data stored in the %_content and %_docsize tables.
*/
#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
typedef struct Fts5Storage Fts5Storage;
int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
int sqlite3Fts5StorageSync(Fts5Storage *p);
int sqlite3Fts5StorageRollback(Fts5Storage *p);
int sqlite3Fts5StorageConfigValue(
Fts5Storage *p, const char*, sqlite3_value*, int
);
int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_expr.c.
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5ExprNode Fts5ExprNode;
typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;
struct Fts5Token {
const char *p; /* Token text (not NULL terminated) */
int n; /* Size of buffer p in bytes */
};
/* Parse a MATCH expression. */
int sqlite3Fts5ExprNew(
Fts5Config *pConfig,
int iCol, /* Column on LHS of MATCH operator */
const char *zExpr,
Fts5Expr **ppNew,
char **pzErr
);
/*
** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
** rc = sqlite3Fts5ExprNext(pExpr)
** ){
** // The document with rowid iRowid matches the expression!
** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
** }
*/
int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
int sqlite3Fts5ExprEof(Fts5Expr*);
i64 sqlite3Fts5ExprRowid(Fts5Expr*);
void sqlite3Fts5ExprFree(Fts5Expr*);
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
int sqlite3Fts5ExprPopulatePoslists(
Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y. */
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
Fts5ExprNode *sqlite3Fts5ParseNode(
Fts5Parse *pParse,
int eType,
Fts5ExprNode *pLeft,
Fts5ExprNode *pRight,
Fts5ExprNearset *pNear
);
Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
Fts5Parse *pParse,
Fts5ExprNode *pLeft,
Fts5ExprNode *pRight
);
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse,
Fts5ExprPhrase *pPhrase,
Fts5Token *pToken,
int bPrefix
);
void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse*,
Fts5ExprNearset*,
Fts5ExprPhrase*
);
Fts5Colset *sqlite3Fts5ParseColset(
Fts5Parse*,
Fts5Colset*,
Fts5Token *
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_aux.c.
*/
int sqlite3Fts5AuxInit(fts5_api*);
/*
** End of interface to code in fts5_aux.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_tokenizer.c.
*/
int sqlite3Fts5TokenizerInit(fts5_api*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_vocab.c.
*/
int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
/*
** End of interface to code in fts5_vocab.c.
**************************************************************************/
/**************************************************************************
** Interface to automatically generated code in fts5_unicode2.c.
*/
int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/
#endif
-714
View File
@@ -1,714 +0,0 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5Int.h"
#include <math.h> /* amalgamator: keep */
/*
** Object used to iterate through all "coalesced phrase instances" in
** a single column of the current row. If the phrase instances in the
** column being considered do not overlap, this object simply iterates
** through them. Or, if they do overlap (share one or more tokens in
** common), each set of overlapping instances is treated as a single
** match. See documentation for the highlight() auxiliary function for
** details.
**
** Usage is:
**
** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter);
** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter);
** rc = fts5CInstIterNext(&iter)
** ){
** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd);
** }
**
*/
typedef struct CInstIter CInstIter;
struct CInstIter {
const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
Fts5Context *pFts; /* First arg to pass to pApi functions */
int iCol; /* Column to search */
int iInst; /* Next phrase instance index */
int nInst; /* Total number of phrase instances */
/* Output variables */
int iStart; /* First token in coalesced phrase instance */
int iEnd; /* Last token in coalesced phrase instance */
};
/*
** Advance the iterator to the next coalesced phrase instance. Return
** an SQLite error code if an error occurs, or SQLITE_OK otherwise.
*/
static int fts5CInstIterNext(CInstIter *pIter){
int rc = SQLITE_OK;
pIter->iStart = -1;
pIter->iEnd = -1;
while( rc==SQLITE_OK && pIter->iInst<pIter->nInst ){
int ip; int ic; int io;
rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io);
if( rc==SQLITE_OK ){
if( ic==pIter->iCol ){
int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip);
if( pIter->iStart<0 ){
pIter->iStart = io;
pIter->iEnd = iEnd;
}else if( io<=pIter->iEnd ){
if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd;
}else{
break;
}
}
pIter->iInst++;
}
}
return rc;
}
/*
** Initialize the iterator object indicated by the final parameter to
** iterate through coalesced phrase instances in column iCol.
*/
static int fts5CInstIterInit(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
int iCol,
CInstIter *pIter
){
int rc;
memset(pIter, 0, sizeof(CInstIter));
pIter->pApi = pApi;
pIter->pFts = pFts;
pIter->iCol = iCol;
rc = pApi->xInstCount(pFts, &pIter->nInst);
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(pIter);
}
return rc;
}
/*************************************************************************
** Start of highlight() implementation.
*/
typedef struct HighlightContext HighlightContext;
struct HighlightContext {
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
int iRangeStart; /* First token to include */
int iRangeEnd; /* If non-zero, last token to include */
const char *zOpen; /* Opening highlight */
const char *zClose; /* Closing highlight */
const char *zIn; /* Input text */
int nIn; /* Size of input text in bytes */
int iOff; /* Current offset within zIn[] */
char *zOut; /* Output value */
};
/*
** Append text to the HighlightContext output string - p->zOut. Argument
** z points to a buffer containing n bytes of text to append. If n is
** negative, everything up until the first '\0' is appended to the output.
**
** If *pRc is set to any value other than SQLITE_OK when this function is
** called, it is a no-op. If an error (i.e. an OOM condition) is encountered,
** *pRc is set to an error code before returning.
*/
static void fts5HighlightAppend(
int *pRc,
HighlightContext *p,
const char *z, int n
){
if( *pRc==SQLITE_OK ){
if( n<0 ) n = (int)strlen(z);
p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
}
}
/*
** Tokenizer callback used by implementation of highlight() function.
*/
static int fts5HighlightCb(
void *pContext, /* Pointer to HighlightContext object */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
int iPos;
UNUSED_PARAM2(pToken, nToken);
if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
iPos = p->iPos++;
if( p->iRangeEnd>0 ){
if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
}
if( iPos==p->iter.iStart ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->iOff = iStartOff;
}
if( iPos==p->iter.iEnd ){
if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
fts5HighlightAppend(&rc, p, p->zOpen, -1);
}
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->iOff = iEndOff;
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(&p->iter);
}
}
if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
fts5HighlightAppend(&rc, p, p->zClose, -1);
}
}
return rc;
}
/*
** Implementation of highlight() function.
*/
static void fts5HighlightFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
HighlightContext ctx;
int rc;
int iCol;
if( nVal!=3 ){
const char *zErr = "wrong number of arguments to function highlight()";
sqlite3_result_error(pCtx, zErr, -1);
return;
}
iCol = sqlite3_value_int(apVal[0]);
memset(&ctx, 0, sizeof(HighlightContext));
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}
sqlite3_free(ctx.zOut);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}
}
/*
** End of highlight() implementation.
**************************************************************************/
/*
** Context object passed to the fts5SentenceFinderCb() function.
*/
typedef struct Fts5SFinder Fts5SFinder;
struct Fts5SFinder {
int iPos; /* Current token position */
int nFirstAlloc; /* Allocated size of aFirst[] */
int nFirst; /* Number of entries in aFirst[] */
int *aFirst; /* Array of first token in each sentence */
const char *zDoc; /* Document being tokenized */
};
/*
** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
** error occurs.
*/
static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
if( p->nFirstAlloc==p->nFirst ){
int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
int *aNew;
aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
if( aNew==0 ) return SQLITE_NOMEM;
p->aFirst = aNew;
p->nFirstAlloc = nNew;
}
p->aFirst[p->nFirst++] = iAdd;
return SQLITE_OK;
}
/*
** This function is an xTokenize() callback used by the auxiliary snippet()
** function. Its job is to identify tokens that are the first in a sentence.
** For each such token, an entry is added to the SFinder.aFirst[] array.
*/
static int fts5SentenceFinderCb(
void *pContext, /* Pointer to HighlightContext object */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
){
int rc = SQLITE_OK;
UNUSED_PARAM2(pToken, nToken);
UNUSED_PARAM(iEndOff);
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
Fts5SFinder *p = (Fts5SFinder*)pContext;
if( p->iPos>0 ){
int i;
char c = 0;
for(i=iStartOff-1; i>=0; i--){
c = p->zDoc[i];
if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
}
if( i!=iStartOff-1 && (c=='.' || c==':') ){
rc = fts5SentenceFinderAdd(p, p->iPos);
}
}else{
rc = fts5SentenceFinderAdd(p, 0);
}
p->iPos++;
}
return rc;
}
static int fts5SnippetScore(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
int nDocsize, /* Size of column in tokens */
unsigned char *aSeen, /* Array with one element per query phrase */
int iCol, /* Column to score */
int iPos, /* Starting offset to score */
int nToken, /* Max tokens per snippet */
int *pnScore, /* OUT: Score */
int *piPos /* OUT: Adjusted offset */
){
int rc;
int i;
int ip = 0;
int ic = 0;
int iOff = 0;
int iFirst = -1;
int nInst;
int nScore = 0;
int iLast = 0;
rc = pApi->xInstCount(pFts, &nInst);
for(i=0; i<nInst && rc==SQLITE_OK; i++){
rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
nScore += (aSeen[ip] ? 1 : 1000);
aSeen[ip] = 1;
if( iFirst<0 ) iFirst = iOff;
iLast = iOff + pApi->xPhraseSize(pFts, ip);
}
}
*pnScore = nScore;
if( piPos ){
int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
if( iAdj<0 ) iAdj = 0;
*piPos = iAdj;
}
return rc;
}
/*
** Return the value in pVal interpreted as utf-8 text. Except, if pVal
** contains a NULL value, return a pointer to a static string zero
** bytes in length instead of a NULL pointer.
*/
static const char *fts5ValueToText(sqlite3_value *pVal){
const char *zRet = (const char*)sqlite3_value_text(pVal);
return zRet ? zRet : "";
}
/*
** Implementation of snippet() function.
*/
static void fts5SnippetFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
HighlightContext ctx;
int rc = SQLITE_OK; /* Return code */
int iCol; /* 1st argument to snippet() */
const char *zEllips; /* 4th argument to snippet() */
int nToken; /* 5th argument to snippet() */
int nInst = 0; /* Number of instance matches this row */
int i; /* Used to iterate through instances */
int nPhrase; /* Number of phrases in query */
unsigned char *aSeen; /* Array of "seen instance" flags */
int iBestCol; /* Column containing best snippet */
int iBestStart = 0; /* First token of best snippet */
int nBestScore = 0; /* Score of best snippet */
int nColSize = 0; /* Total size of iBestCol in tokens */
Fts5SFinder sFinder; /* Used to find the beginnings of sentences */
int nCol;
if( nVal!=5 ){
const char *zErr = "wrong number of arguments to function snippet()";
sqlite3_result_error(pCtx, zErr, -1);
return;
}
nCol = pApi->xColumnCount(pFts);
memset(&ctx, 0, sizeof(HighlightContext));
iCol = sqlite3_value_int(apVal[0]);
ctx.zOpen = fts5ValueToText(apVal[1]);
ctx.zClose = fts5ValueToText(apVal[2]);
zEllips = fts5ValueToText(apVal[3]);
nToken = sqlite3_value_int(apVal[4]);
iBestCol = (iCol>=0 ? iCol : 0);
nPhrase = pApi->xPhraseCount(pFts);
aSeen = sqlite3_malloc(nPhrase);
if( aSeen==0 ){
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
rc = pApi->xInstCount(pFts, &nInst);
}
memset(&sFinder, 0, sizeof(Fts5SFinder));
for(i=0; i<nCol; i++){
if( iCol<0 || iCol==i ){
int nDoc;
int nDocsize;
int ii;
sFinder.iPos = 0;
sFinder.nFirst = 0;
rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xTokenize(pFts,
sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
);
if( rc!=SQLITE_OK ) break;
rc = pApi->xColumnSize(pFts, i, &nDocsize);
if( rc!=SQLITE_OK ) break;
for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
int ip, ic, io;
int iAdj;
int nScore;
int jj;
rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
if( ic!=i || rc!=SQLITE_OK ) continue;
memset(aSeen, 0, nPhrase);
rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
io, nToken, &nScore, &iAdj
);
if( rc==SQLITE_OK && nScore>nBestScore ){
nBestScore = nScore;
iBestCol = i;
iBestStart = iAdj;
nColSize = nDocsize;
}
if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
for(jj=0; jj<(sFinder.nFirst-1); jj++){
if( sFinder.aFirst[jj+1]>io ) break;
}
if( sFinder.aFirst[jj]<io ){
memset(aSeen, 0, nPhrase);
rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
sFinder.aFirst[jj], nToken, &nScore, 0
);
nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
if( rc==SQLITE_OK && nScore>nBestScore ){
nBestScore = nScore;
iBestCol = i;
iBestStart = sFinder.aFirst[jj];
nColSize = nDocsize;
}
}
}
}
}
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
}
if( rc==SQLITE_OK && nColSize==0 ){
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
}
ctx.iRangeStart = iBestStart;
ctx.iRangeEnd = iBestStart + nToken - 1;
if( iBestStart>0 ){
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
/* Advance iterator ctx.iter so that it points to the first coalesced
** phrase instance at or following position iBestStart. */
while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){
rc = fts5CInstIterNext(&ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
if( ctx.iRangeEnd>=(nColSize-1) ){
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
}else{
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3_free(ctx.zOut);
sqlite3_free(aSeen);
sqlite3_free(sFinder.aFirst);
}
/************************************************************************/
/*
** The first time the bm25() function is called for a query, an instance
** of the following structure is allocated and populated.
*/
typedef struct Fts5Bm25Data Fts5Bm25Data;
struct Fts5Bm25Data {
int nPhrase; /* Number of phrases in query */
double avgdl; /* Average number of tokens in each row */
double *aIDF; /* IDF for each phrase */
double *aFreq; /* Array used to calculate phrase freq. */
};
/*
** Callback used by fts5Bm25GetData() to count the number of rows in the
** table matched by each individual phrase within the query.
*/
static int fts5CountCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
void *pUserData /* Pointer to sqlite3_int64 variable */
){
sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
UNUSED_PARAM2(pApi, pFts);
(*pn)++;
return SQLITE_OK;
}
/*
** Set *ppData to point to the Fts5Bm25Data object for the current query.
** If the object has not already been allocated, allocate and populate it
** now.
*/
static int fts5Bm25GetData(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */
){
int rc = SQLITE_OK; /* Return code */
Fts5Bm25Data *p; /* Object to return */
p = pApi->xGetAuxdata(pFts, 0);
if( p==0 ){
int nPhrase; /* Number of phrases in query */
sqlite3_int64 nRow = 0; /* Number of rows in table */
sqlite3_int64 nToken = 0; /* Number of tokens in table */
int nByte; /* Bytes of space to allocate */
int i;
/* Allocate the Fts5Bm25Data object */
nPhrase = pApi->xPhraseCount(pFts);
nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
p = (Fts5Bm25Data*)sqlite3_malloc(nByte);
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
memset(p, 0, nByte);
p->nPhrase = nPhrase;
p->aIDF = (double*)&p[1];
p->aFreq = &p->aIDF[nPhrase];
}
/* Calculate the average document length for this FTS5 table */
if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow);
if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow;
/* Calculate an IDF for each phrase in the query */
for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
sqlite3_int64 nHit = 0;
rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb);
if( rc==SQLITE_OK ){
/* Calculate the IDF (Inverse Document Frequency) for phrase i.
** This is done using the standard BM25 formula as found on wikipedia:
**
** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
**
** where "N" is the total number of documents in the set and nHit
** is the number that contain at least one instance of the phrase
** under consideration.
**
** The problem with this is that if (N < 2*nHit), the IDF is
** negative. Which is undesirable. So the mimimum allowable IDF is
** (1e-6) - roughly the same as a term that appears in just over
** half of set of 5,000,000 documents. */
double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) );
if( idf<=0.0 ) idf = 1e-6;
p->aIDF[i] = idf;
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(p);
}else{
rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
}
if( rc!=SQLITE_OK ) p = 0;
}
*ppData = p;
return rc;
}
/*
** Implementation of bm25() function.
*/
static void fts5Bm25Function(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
const double k1 = 1.2; /* Constant "k1" from BM25 formula */
const double b = 0.75; /* Constant "b" from BM25 formula */
int rc = SQLITE_OK; /* Error code */
double score = 0.0; /* SQL function return value */
Fts5Bm25Data *pData; /* Values allocated/calculated once only */
int i; /* Iterator variable */
int nInst = 0; /* Value returned by xInstCount() */
double D = 0.0; /* Total number of tokens in row */
double *aFreq = 0; /* Array of phrase freq. for current row */
/* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation)
** for each phrase in the query for the current row. */
rc = fts5Bm25GetData(pApi, pFts, &pData);
if( rc==SQLITE_OK ){
aFreq = pData->aFreq;
memset(aFreq, 0, sizeof(double) * pData->nPhrase);
rc = pApi->xInstCount(pFts, &nInst);
}
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int ip; int ic; int io;
rc = pApi->xInst(pFts, i, &ip, &ic, &io);
if( rc==SQLITE_OK ){
double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0;
aFreq[ip] += w;
}
}
/* Figure out the total size of the current row in tokens. */
if( rc==SQLITE_OK ){
int nTok;
rc = pApi->xColumnSize(pFts, -1, &nTok);
D = (double)nTok;
}
/* Determine the BM25 score for the current row. */
for(i=0; rc==SQLITE_OK && i<pData->nPhrase; i++){
score += pData->aIDF[i] * (
( aFreq[i] * (k1 + 1.0) ) /
( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) )
);
}
/* If no error has occurred, return the calculated score. Otherwise,
** throw an SQL exception. */
if( rc==SQLITE_OK ){
sqlite3_result_double(pCtx, -1.0 * score);
}else{
sqlite3_result_error_code(pCtx, rc);
}
}
int sqlite3Fts5AuxInit(fts5_api *pApi){
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
void *pUserData; /* User-data pointer */
fts5_extension_function xFunc;/* Callback function */
void (*xDestroy)(void*); /* Destructor function */
} aBuiltin [] = {
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
rc = pApi->xCreateFunction(pApi,
aBuiltin[i].zFunc,
aBuiltin[i].pUserData,
aBuiltin[i].xFunc,
aBuiltin[i].xDestroy
);
}
return rc;
}
-394
View File
@@ -1,394 +0,0 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5Int.h"
int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
if( (u32)pBuf->nSpace<nByte ){
u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
u8 *pNew;
while( nNew<nByte ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
}
}
return 0;
}
/*
** Encode value iVal as an SQLite varint and append it to the buffer object
** pBuf. If an OOM error occurs, set the error code in p.
*/
void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
if( fts5BufferGrow(pRc, pBuf, 9) ) return;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
}
void sqlite3Fts5Put32(u8 *aBuf, int iVal){
aBuf[0] = (iVal>>24) & 0x00FF;
aBuf[1] = (iVal>>16) & 0x00FF;
aBuf[2] = (iVal>> 8) & 0x00FF;
aBuf[3] = (iVal>> 0) & 0x00FF;
}
int sqlite3Fts5Get32(const u8 *aBuf){
return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
}
/*
** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferAppendBlob(
int *pRc,
Fts5Buffer *pBuf,
u32 nData,
const u8 *pData
){
assert_nc( *pRc || nData>=0 );
if( nData ){
if( fts5BufferGrow(pRc, pBuf, nData) ) return;
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
}
/*
** Append the nul-terminated string zStr to the buffer pBuf. This function
** ensures that the byte following the buffer data is set to 0x00, even
** though this byte is not included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendString(
int *pRc,
Fts5Buffer *pBuf,
const char *zStr
){
int nStr = (int)strlen(zStr);
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
pBuf->n--;
}
/*
** Argument zFmt is a printf() style format string. This function performs
** the printf() style processing, then appends the results to buffer pBuf.
**
** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
** following the buffer data is set to 0x00, even though this byte is not
** included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendPrintf(
int *pRc,
Fts5Buffer *pBuf,
char *zFmt, ...
){
if( *pRc==SQLITE_OK ){
char *zTmp;
va_list ap;
va_start(ap, zFmt);
zTmp = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zTmp==0 ){
*pRc = SQLITE_NOMEM;
}else{
sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
sqlite3_free(zTmp);
}
}
}
char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
va_list ap;
va_start(ap, zFmt);
zRet = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zRet==0 ){
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
/*
** Free any buffer allocated by pBuf. Zero the structure before returning.
*/
void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
sqlite3_free(pBuf->p);
memset(pBuf, 0, sizeof(Fts5Buffer));
}
/*
** Zero the contents of the buffer object. But do not free the associated
** memory allocation.
*/
void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
pBuf->n = 0;
}
/*
** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferSet(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
pBuf->n = 0;
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
}
int sqlite3Fts5PoslistNext64(
const u8 *a, int n, /* Buffer containing poslist */
int *pi, /* IN/OUT: Offset within a[] */
i64 *piOff /* IN/OUT: Current offset */
){
int i = *pi;
if( i>=n ){
/* EOF */
*piOff = -1;
return 1;
}else{
i64 iOff = *piOff;
int iVal;
fts5FastGetVarint32(a, i, iVal);
if( iVal==1 ){
fts5FastGetVarint32(a, i, iVal);
iOff = ((i64)iVal) << 32;
fts5FastGetVarint32(a, i, iVal);
}
*piOff = iOff + (iVal-2);
*pi = i;
return 0;
}
}
/*
** Advance the iterator object passed as the only argument. Return true
** if the iterator reaches EOF, or false otherwise.
*/
int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){
pIter->bEof = 1;
}
return pIter->bEof;
}
int sqlite3Fts5PoslistReaderInit(
const u8 *a, int n, /* Poslist buffer to iterate through */
Fts5PoslistReader *pIter /* Iterator object to initialize */
){
memset(pIter, 0, sizeof(*pIter));
pIter->a = a;
pIter->n = n;
sqlite3Fts5PoslistReaderNext(pIter);
return pIter->bEof;
}
/*
** Append position iPos to the position list being accumulated in buffer
** pBuf, which must be already be large enough to hold the new data.
** The previous position written to this list is *piPrev. *piPrev is set
** to iPos before returning.
*/
void sqlite3Fts5PoslistSafeAppend(
Fts5Buffer *pBuf,
i64 *piPrev,
i64 iPos
){
static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
if( (iPos & colmask) != (*piPrev & colmask) ){
pBuf->p[pBuf->n++] = 1;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
*piPrev = (iPos & colmask);
}
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
*piPrev = iPos;
}
int sqlite3Fts5PoslistWriterAppend(
Fts5Buffer *pBuf,
Fts5PoslistWriter *pWriter,
i64 iPos
){
int rc = 0; /* Initialized only to suppress erroneous warning from Clang */
if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
return SQLITE_OK;
}
void *sqlite3Fts5MallocZero(int *pRc, int nByte){
void *pRet = 0;
if( *pRc==SQLITE_OK ){
pRet = sqlite3_malloc(nByte);
if( pRet==0 ){
if( nByte>0 ) *pRc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
}
}
return pRet;
}
/*
** Return a nul-terminated copy of the string indicated by pIn. If nIn
** is non-negative, then it is the length of the string in bytes. Otherwise,
** the length of the string is determined using strlen().
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
if( nIn<0 ){
nIn = (int)strlen(pIn);
}
zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
}else{
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
/*
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
** * All non-ASCII characters,
** * The 52 upper and lower case ASCII characters, and
** * The 10 integer ASCII characters.
** * The underscore character "_" (0x5F).
** * The unicode "subsitute" character (0x1A).
*/
int sqlite3Fts5IsBareword(char t){
u8 aBareword[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
};
return (t & 0x80) || aBareword[(int)t];
}
/*************************************************************************
*/
typedef struct Fts5TermsetEntry Fts5TermsetEntry;
struct Fts5TermsetEntry {
char *pTerm;
int nTerm;
int iIdx; /* Index (main or aPrefix[] entry) */
Fts5TermsetEntry *pNext;
};
struct Fts5Termset {
Fts5TermsetEntry *apHash[512];
};
int sqlite3Fts5TermsetNew(Fts5Termset **pp){
int rc = SQLITE_OK;
*pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
return rc;
}
int sqlite3Fts5TermsetAdd(
Fts5Termset *p,
int iIdx,
const char *pTerm, int nTerm,
int *pbPresent
){
int rc = SQLITE_OK;
*pbPresent = 0;
if( p ){
int i;
u32 hash = 13;
Fts5TermsetEntry *pEntry;
/* Calculate a hash value for this term. This is the same hash checksum
** used by the fts5_hash.c module. This is not important for correct
** operation of the module, but is necessary to ensure that some tests
** designed to produce hash table collisions really do work. */
for(i=nTerm-1; i>=0; i--){
hash = (hash << 3) ^ hash ^ pTerm[i];
}
hash = (hash << 3) ^ hash ^ iIdx;
hash = hash % ArraySize(p->apHash);
for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
if( pEntry->iIdx==iIdx
&& pEntry->nTerm==nTerm
&& memcmp(pEntry->pTerm, pTerm, nTerm)==0
){
*pbPresent = 1;
break;
}
}
if( pEntry==0 ){
pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
if( pEntry ){
pEntry->pTerm = (char*)&pEntry[1];
pEntry->nTerm = nTerm;
pEntry->iIdx = iIdx;
memcpy(pEntry->pTerm, pTerm, nTerm);
pEntry->pNext = p->apHash[hash];
p->apHash[hash] = pEntry;
}
}
}
return rc;
}
void sqlite3Fts5TermsetFree(Fts5Termset *p){
if( p ){
u32 i;
for(i=0; i<ArraySize(p->apHash); i++){
Fts5TermsetEntry *pEntry = p->apHash[i];
while( pEntry ){
Fts5TermsetEntry *pDel = pEntry;
pEntry = pEntry->pNext;
sqlite3_free(pDel);
}
}
sqlite3_free(p);
}
}
-961
View File
@@ -1,961 +0,0 @@
/*
** 2014 Jun 09
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/
#include "fts5Int.h"
#define FTS5_DEFAULT_PAGE_SIZE 4050
#define FTS5_DEFAULT_AUTOMERGE 4
#define FTS5_DEFAULT_USERMERGE 4
#define FTS5_DEFAULT_CRISISMERGE 16
#define FTS5_DEFAULT_HASHSIZE (1024*1024)
/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (128*1024)
static int fts5_iswhitespace(char x){
return (x==' ');
}
static int fts5_isopenquote(char x){
return (x=='"' || x=='\'' || x=='[' || x=='`');
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a white-space character.
*/
static const char *fts5ConfigSkipWhitespace(const char *pIn){
const char *p = pIn;
if( p ){
while( fts5_iswhitespace(*p) ){ p++; }
}
return p;
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
const char *p = pIn;
while ( sqlite3Fts5IsBareword(*p) ) p++;
if( p==pIn ) p = 0;
return p;
}
static int fts5_isdigit(char a){
return (a>='0' && a<='9');
}
static const char *fts5ConfigSkipLiteral(const char *pIn){
const char *p = pIn;
switch( *p ){
case 'n': case 'N':
if( sqlite3_strnicmp("null", p, 4)==0 ){
p = &p[4];
}else{
p = 0;
}
break;
case 'x': case 'X':
p++;
if( *p=='\'' ){
p++;
while( (*p>='a' && *p<='f')
|| (*p>='A' && *p<='F')
|| (*p>='0' && *p<='9')
){
p++;
}
if( *p=='\'' && 0==((p-pIn)%2) ){
p++;
}else{
p = 0;
}
}else{
p = 0;
}
break;
case '\'':
p++;
while( p ){
if( *p=='\'' ){
p++;
if( *p!='\'' ) break;
}
p++;
if( *p==0 ) p = 0;
}
break;
default:
/* maybe a number */
if( *p=='+' || *p=='-' ) p++;
while( fts5_isdigit(*p) ) p++;
/* At this point, if the literal was an integer, the parse is
** finished. Or, if it is a floating point value, it may continue
** with either a decimal point or an 'E' character. */
if( *p=='.' && fts5_isdigit(p[1]) ){
p += 2;
while( fts5_isdigit(*p) ) p++;
}
if( p==pIn ) p = 0;
break;
}
return p;
}
/*
** The first character of the string pointed to by argument z is guaranteed
** to be an open-quote character (see function fts5_isopenquote()).
**
** This function searches for the corresponding close-quote character within
** the string and, if found, dequotes the string in place and adds a new
** nul-terminator byte.
**
** If the close-quote is found, the value returned is the byte offset of
** the character immediately following it. Or, if the close-quote is not
** found, -1 is returned. If -1 is returned, the buffer is left in an
** undefined state.
*/
static int fts5Dequote(char *z){
char q;
int iIn = 1;
int iOut = 0;
q = z[0];
/* Set stack variable q to the close-quote character */
assert( q=='[' || q=='\'' || q=='"' || q=='`' );
if( q=='[' ) q = ']';
while( ALWAYS(z[iIn]) ){
if( z[iIn]==q ){
if( z[iIn+1]!=q ){
/* Character iIn was the close quote. */
iIn++;
break;
}else{
/* Character iIn and iIn+1 form an escaped quote character. Skip
** the input cursor past both and copy a single quote character
** to the output buffer. */
iIn += 2;
z[iOut++] = q;
}
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
return iIn;
}
/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters. The conversion is done in-place. If the
** input does not begin with a quote character, then this routine
** is a no-op.
**
** Examples:
**
** "abc" becomes abc
** 'xyz' becomes xyz
** [pqr] becomes pqr
** `mno` becomes mno
*/
void sqlite3Fts5Dequote(char *z){
char quote; /* Quote character (if any ) */
assert( 0==fts5_iswhitespace(z[0]) );
quote = z[0];
if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
fts5Dequote(z);
}
}
struct Fts5Enum {
const char *zName;
int eVal;
};
typedef struct Fts5Enum Fts5Enum;
static int fts5ConfigSetEnum(
const Fts5Enum *aEnum,
const char *zEnum,
int *peVal
){
int nEnum = (int)strlen(zEnum);
int i;
int iVal = -1;
for(i=0; aEnum[i].zName; i++){
if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
if( iVal>=0 ) return SQLITE_ERROR;
iVal = aEnum[i].eVal;
}
}
*peVal = iVal;
return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
}
/*
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Global *pGlobal,
Fts5Config *pConfig, /* Configuration object to update */
const char *zCmd, /* Special command to parse */
const char *zArg, /* Argument to parse */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK;
int nCmd = (int)strlen(zCmd);
if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
const char *p;
int bFirst = 1;
if( pConfig->aPrefix==0 ){
pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
if( rc ) return rc;
}
p = zArg;
while( 1 ){
int nPre = 0;
while( p[0]==' ' ) p++;
if( bFirst==0 && p[0]==',' ){
p++;
while( p[0]==' ' ) p++;
}else if( p[0]=='\0' ){
break;
}
if( p[0]<'0' || p[0]>'9' ){
*pzErr = sqlite3_mprintf("malformed prefix=... directive");
rc = SQLITE_ERROR;
break;
}
if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
*pzErr = sqlite3_mprintf(
"too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
);
rc = SQLITE_ERROR;
break;
}
while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
nPre = nPre*10 + (p[0] - '0');
p++;
}
if( nPre<=0 || nPre>=1000 ){
*pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
rc = SQLITE_ERROR;
break;
}
pConfig->aPrefix[pConfig->nPrefix] = nPre;
pConfig->nPrefix++;
bFirst = 0;
}
assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
return rc;
}
if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
const char *p = (const char*)zArg;
int nArg = (int)strlen(zArg) + 1;
char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
char *pSpace = pDel;
if( azArg && pSpace ){
if( pConfig->pTok ){
*pzErr = sqlite3_mprintf("multiple tokenize=... directives");
rc = SQLITE_ERROR;
}else{
for(nArg=0; p && *p; nArg++){
const char *p2 = fts5ConfigSkipWhitespace(p);
if( *p2=='\'' ){
p = fts5ConfigSkipLiteral(p2);
}else{
p = fts5ConfigSkipBareword(p2);
}
if( p ){
memcpy(pSpace, p2, p-p2);
azArg[nArg] = pSpace;
sqlite3Fts5Dequote(pSpace);
pSpace += (p - p2) + 1;
p = fts5ConfigSkipWhitespace(p);
}
}
if( p==0 ){
*pzErr = sqlite3_mprintf("parse error in tokenize directive");
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
pzErr
);
}
}
}
sqlite3_free(azArg);
sqlite3_free(pDel);
return rc;
}
if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
*pzErr = sqlite3_mprintf("multiple content=... directives");
rc = SQLITE_ERROR;
}else{
if( zArg[0] ){
pConfig->eContent = FTS5_CONTENT_EXTERNAL;
pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
}else{
pConfig->eContent = FTS5_CONTENT_NONE;
}
}
return rc;
}
if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
if( pConfig->zContentRowid ){
*pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
rc = SQLITE_ERROR;
}else{
pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
}
return rc;
}
if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed columnsize=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bColumnsize = (zArg[0]=='1');
}
return rc;
}
if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
const Fts5Enum aDetail[] = {
{ "none", FTS5_DETAIL_NONE },
{ "full", FTS5_DETAIL_FULL },
{ "columns", FTS5_DETAIL_COLUMNS },
{ 0, 0 }
};
if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
*pzErr = sqlite3_mprintf("malformed detail=... directive");
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
return SQLITE_ERROR;
}
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
);
}
/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
** to find close-quote character).
**
** Before returning, set pzOut to point to a new buffer containing a
** nul-terminated, dequoted copy of the gobbled word. If the word was
** quoted, *pbQuoted is also set to 1 before returning.
**
** If *pRc is other than SQLITE_OK when this function is called, it is
** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
** set if a parse error (failed to find close quote) occurs.
*/
static const char *fts5ConfigGobbleWord(
int *pRc, /* IN/OUT: Error code */
const char *zIn, /* Buffer to gobble string/bareword from */
char **pzOut, /* OUT: malloc'd buffer containing str/bw */
int *pbQuoted /* OUT: Set to true if dequoting required */
){
const char *zRet = 0;
int nIn = (int)strlen(zIn);
char *zOut = sqlite3_malloc(nIn+1);
assert( *pRc==SQLITE_OK );
*pbQuoted = 0;
*pzOut = 0;
if( zOut==0 ){
*pRc = SQLITE_NOMEM;
}else{
memcpy(zOut, zIn, nIn+1);
if( fts5_isopenquote(zOut[0]) ){
int ii = fts5Dequote(zOut);
zRet = &zIn[ii];
*pbQuoted = 1;
}else{
zRet = fts5ConfigSkipBareword(zIn);
if( zRet ){
zOut[zRet-zIn] = '\0';
}
}
}
if( zRet==0 ){
sqlite3_free(zOut);
}else{
*pzOut = zOut;
}
return zRet;
}
static int fts5ConfigParseColumn(
Fts5Config *p,
char *zCol,
char *zArg,
char **pzErr
){
int rc = SQLITE_OK;
if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME)
|| 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME)
){
*pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
rc = SQLITE_ERROR;
}else if( zArg ){
if( 0==sqlite3_stricmp(zArg, "unindexed") ){
p->abUnindexed[p->nCol] = 1;
}else{
*pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
rc = SQLITE_ERROR;
}
}
p->azCol[p->nCol++] = zCol;
return rc;
}
/*
** Populate the Fts5Config.zContentExprlist string.
*/
static int fts5ConfigMakeExprlist(Fts5Config *p){
int i;
int rc = SQLITE_OK;
Fts5Buffer buf = {0, 0, 0};
sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
if( p->eContent!=FTS5_CONTENT_NONE ){
for(i=0; i<p->nCol; i++){
if( p->eContent==FTS5_CONTENT_EXTERNAL ){
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
}else{
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
}
}
}
assert( p->zContentExprlist==0 );
p->zContentExprlist = (char*)buf.p;
return rc;
}
/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to
** allocate an instance of Fts5Config containing the results of parsing
** those arguments.
**
** If successful, SQLITE_OK is returned and *ppOut is set to point to the
** new Fts5Config object. If an error occurs, an SQLite error code is
** returned, *ppOut is set to NULL and an error message may be left in
** *pzErr. It is the responsibility of the caller to eventually free any
** such error message using sqlite3_free().
*/
int sqlite3Fts5ConfigParse(
Fts5Global *pGlobal,
sqlite3 *db,
int nArg, /* Number of arguments */
const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
Fts5Config **ppOut, /* OUT: Results of parse */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK; /* Return code */
Fts5Config *pRet; /* New object to return */
int i;
int nByte;
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
memset(pRet, 0, sizeof(Fts5Config));
pRet->db = db;
pRet->iCookie = -1;
nByte = nArg * (sizeof(char*) + sizeof(u8));
pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
pRet->bColumnsize = 1;
pRet->eDetail = FTS5_DETAIL_FULL;
#ifdef SQLITE_DEBUG
pRet->bPrefixIndex = 1;
#endif
if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
rc = SQLITE_ERROR;
}
for(i=3; rc==SQLITE_OK && i<nArg; i++){
const char *zOrig = azArg[i];
const char *z;
char *zOne = 0;
char *zTwo = 0;
int bOption = 0;
int bMustBeCol = 0;
z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
z = fts5ConfigSkipWhitespace(z);
if( z && *z=='=' ){
bOption = 1;
z++;
if( bMustBeCol ) z = 0;
}
z = fts5ConfigSkipWhitespace(z);
if( z && z[0] ){
int bDummy;
z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
if( z && z[0] ) z = 0;
}
if( rc==SQLITE_OK ){
if( z==0 ){
*pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
}else{
rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
zOne = 0;
}
}
}
sqlite3_free(zOne);
sqlite3_free(zTwo);
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
const char *zTail = 0;
assert( pRet->eContent==FTS5_CONTENT_NORMAL
|| pRet->eContent==FTS5_CONTENT_NONE
);
if( pRet->eContent==FTS5_CONTENT_NORMAL ){
zTail = "content";
}else if( pRet->bColumnsize ){
zTail = "docsize";
}
if( zTail ){
pRet->zContent = sqlite3Fts5Mprintf(
&rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
);
}
}
if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
}
/* Formulate the zContentExprlist text */
if( rc==SQLITE_OK ){
rc = fts5ConfigMakeExprlist(pRet);
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
}
return rc;
}
/*
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->pTok ){
pConfig->pTokApi->xDelete(pConfig->pTok);
}
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
sqlite3_free(pConfig->azCol[i]);
}
sqlite3_free(pConfig->azCol);
sqlite3_free(pConfig->aPrefix);
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
sqlite3_free(pConfig->zContent);
sqlite3_free(pConfig->zContentRowid);
sqlite3_free(pConfig->zContentExprlist);
sqlite3_free(pConfig);
}
}
/*
** Call sqlite3_declare_vtab() based on the contents of the configuration
** object passed as the only argument. Return SQLITE_OK if successful, or
** an SQLite error code if an error occurs.
*/
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
int i;
int rc = SQLITE_OK;
char *zSql;
zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
for(i=0; zSql && i<pConfig->nCol; i++){
const char *zSep = (i==0?"":", ");
zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
}
zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
zSql, pConfig->zName, FTS5_RANK_NAME
);
assert( zSql || rc==SQLITE_NOMEM );
if( zSql ){
rc = sqlite3_declare_vtab(pConfig->db, zSql);
sqlite3_free(zSql);
}
return rc;
}
/*
** Tokenize the text passed via the second and third arguments.
**
** The callback is invoked once for each token in the input text. The
** arguments passed to it are, in order:
**
** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
** const char *pToken // Pointer to buffer containing token
** int nToken // Size of token in bytes
** int iStart // Byte offset of start of token within input text
** int iEnd // Byte offset of end of token within input text
** int iPos // Position of token in input (first token is 0)
**
** If the callback returns a non-zero value the tokenization is abandoned
** and no further callbacks are issued.
**
** This function returns SQLITE_OK if successful or an SQLite error code
** if an error occurs. If the tokenization was abandoned early because
** the callback returned SQLITE_DONE, this is not an error and this function
** still returns SQLITE_OK. Or, if the tokenization was abandoned early
** because the callback returned another non-zero value, it is assumed
** to be an SQLite error code and returned to the caller.
*/
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
){
if( pText==0 ) return SQLITE_OK;
return pConfig->pTokApi->xTokenize(
pConfig->pTok, pCtx, flags, pText, nText, xToken
);
}
/*
** Argument pIn points to the first character in what is expected to be
** a comma-separated list of SQL literals followed by a ')' character.
** If it actually is this, return a pointer to the ')'. Otherwise, return
** NULL to indicate a parse error.
*/
static const char *fts5ConfigSkipArgs(const char *pIn){
const char *p = pIn;
while( 1 ){
p = fts5ConfigSkipWhitespace(p);
p = fts5ConfigSkipLiteral(p);
p = fts5ConfigSkipWhitespace(p);
if( p==0 || *p==')' ) break;
if( *p!=',' ){
p = 0;
break;
}
p++;
}
return p;
}
/*
** Parameter zIn contains a rank() function specification. The format of
** this is:
**
** + Bareword (function name)
** + Open parenthesis - "("
** + Zero or more SQL literals in a comma separated list
** + Close parenthesis - ")"
*/
int sqlite3Fts5ConfigParseRank(
const char *zIn, /* Input string */
char **pzRank, /* OUT: Rank function name */
char **pzRankArgs /* OUT: Rank function arguments */
){
const char *p = zIn;
const char *pRank;
char *zRank = 0;
char *zRankArgs = 0;
int rc = SQLITE_OK;
*pzRank = 0;
*pzRankArgs = 0;
if( p==0 ){
rc = SQLITE_ERROR;
}else{
p = fts5ConfigSkipWhitespace(p);
pRank = p;
p = fts5ConfigSkipBareword(p);
if( p ){
zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
if( zRank ) memcpy(zRank, pRank, p-pRank);
}else{
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK ){
p = fts5ConfigSkipWhitespace(p);
if( *p!='(' ) rc = SQLITE_ERROR;
p++;
}
if( rc==SQLITE_OK ){
const char *pArgs;
p = fts5ConfigSkipWhitespace(p);
pArgs = p;
if( *p!=')' ){
p = fts5ConfigSkipArgs(p);
if( p==0 ){
rc = SQLITE_ERROR;
}else{
zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
}
}
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(zRank);
assert( zRankArgs==0 );
}else{
*pzRank = zRank;
*pzRankArgs = zRankArgs;
}
return rc;
}
int sqlite3Fts5ConfigSetValue(
Fts5Config *pConfig,
const char *zKey,
sqlite3_value *pVal,
int *pbBadkey
){
int rc = SQLITE_OK;
if( 0==sqlite3_stricmp(zKey, "pgsz") ){
int pgsz = 0;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
pgsz = sqlite3_value_int(pVal);
}
if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
*pbBadkey = 1;
}else{
pConfig->pgsz = pgsz;
}
}
else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
int nHashSize = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nHashSize = sqlite3_value_int(pVal);
}
if( nHashSize<=0 ){
*pbBadkey = 1;
}else{
pConfig->nHashSize = nHashSize;
}
}
else if( 0==sqlite3_stricmp(zKey, "automerge") ){
int nAutomerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nAutomerge = sqlite3_value_int(pVal);
}
if( nAutomerge<0 || nAutomerge>64 ){
*pbBadkey = 1;
}else{
if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
pConfig->nAutomerge = nAutomerge;
}
}
else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
int nUsermerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nUsermerge = sqlite3_value_int(pVal);
}
if( nUsermerge<2 || nUsermerge>16 ){
*pbBadkey = 1;
}else{
pConfig->nUsermerge = nUsermerge;
}
}
else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
int nCrisisMerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nCrisisMerge = sqlite3_value_int(pVal);
}
if( nCrisisMerge<0 ){
*pbBadkey = 1;
}else{
if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nCrisisMerge = nCrisisMerge;
}
}
else if( 0==sqlite3_stricmp(zKey, "rank") ){
const char *zIn = (const char*)sqlite3_value_text(pVal);
char *zRank;
char *zRankArgs;
rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
if( rc==SQLITE_OK ){
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
pConfig->zRank = zRank;
pConfig->zRankArgs = zRankArgs;
}else if( rc==SQLITE_ERROR ){
rc = SQLITE_OK;
*pbBadkey = 1;
}
}else{
*pbBadkey = 1;
}
return rc;
}
/*
** Load the contents of the %_config table into memory.
*/
int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
char *zSql;
sqlite3_stmt *p = 0;
int rc = SQLITE_OK;
int iVersion = 0;
/* Set default values */
pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
if( zSql ){
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
sqlite3_free(zSql);
}
assert( rc==SQLITE_OK || p==0 );
if( rc==SQLITE_OK ){
while( SQLITE_ROW==sqlite3_step(p) ){
const char *zK = (const char*)sqlite3_column_text(p, 0);
sqlite3_value *pVal = sqlite3_column_value(p, 1);
if( 0==sqlite3_stricmp(zK, "version") ){
iVersion = sqlite3_value_int(pVal);
}else{
int bDummy = 0;
sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
}
}
rc = sqlite3_finalize(p);
}
if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
rc = SQLITE_ERROR;
if( pConfig->pzErrmsg ){
assert( 0==*pConfig->pzErrmsg );
*pConfig->pzErrmsg = sqlite3_mprintf(
"invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION
);
}
}
if( rc==SQLITE_OK ){
pConfig->iCookie = iCookie;
}
return rc;
}
-2842
View File
File diff suppressed because it is too large Load Diff
-537
View File
@@ -1,537 +0,0 @@
/*
** 2014 August 11
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#include "fts5Int.h"
typedef struct Fts5HashEntry Fts5HashEntry;
/*
** This file contains the implementation of an in-memory hash table used
** to accumuluate "term -> doclist" content before it is flused to a level-0
** segment.
*/
struct Fts5Hash {
int eDetail; /* Copy of Fts5Config.eDetail */
int *pnByte; /* Pointer to bytes counter */
int nEntry; /* Number of entries currently in hash */
int nSlot; /* Size of aSlot[] array */
Fts5HashEntry *pScan; /* Current ordered scan item */
Fts5HashEntry **aSlot; /* Array of hash slots */
};
/*
** Each entry in the hash table is represented by an object of the
** following type. Each object, its key (a nul-terminated string) and
** its current data are stored in a single memory allocation. The
** key immediately follows the object in memory. The position list
** data immediately follows the key data in memory.
**
** The data that follows the key is in a similar, but not identical format
** to the doclist data stored in the database. It is:
**
** * Rowid, as a varint
** * Position list, without 0x00 terminator.
** * Size of previous position list and rowid, as a 4 byte
** big-endian integer.
**
** iRowidOff:
** Offset of last rowid written to data area. Relative to first byte of
** structure.
**
** nData:
** Bytes of data written since iRowidOff.
*/
struct Fts5HashEntry {
Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */
Fts5HashEntry *pScanNext; /* Next entry in sorted order */
int nAlloc; /* Total size of allocation */
int iSzPoslist; /* Offset of space for 4-byte poslist size */
int nData; /* Total bytes of data (incl. structure) */
int nKey; /* Length of key in bytes */
u8 bDel; /* Set delete-flag @ iSzPoslist */
u8 bContent; /* Set content-flag (detail=none mode) */
i16 iCol; /* Column of last value written */
int iPos; /* Position of last value written */
i64 iRowid; /* Rowid of last value written */
};
/*
** Eqivalent to:
**
** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
*/
#define fts5EntryKey(p) ( ((char *)(&(p)[1])) )
/*
** Allocate a new hash table.
*/
int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
int rc = SQLITE_OK;
Fts5Hash *pNew;
*ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int nByte;
memset(pNew, 0, sizeof(Fts5Hash));
pNew->pnByte = pnByte;
pNew->eDetail = pConfig->eDetail;
pNew->nSlot = 1024;
nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
if( pNew->aSlot==0 ){
sqlite3_free(pNew);
*ppNew = 0;
rc = SQLITE_NOMEM;
}else{
memset(pNew->aSlot, 0, nByte);
}
}
return rc;
}
/*
** Free a hash table object.
*/
void sqlite3Fts5HashFree(Fts5Hash *pHash){
if( pHash ){
sqlite3Fts5HashClear(pHash);
sqlite3_free(pHash->aSlot);
sqlite3_free(pHash);
}
}
/*
** Empty (but do not delete) a hash table.
*/
void sqlite3Fts5HashClear(Fts5Hash *pHash){
int i;
for(i=0; i<pHash->nSlot; i++){
Fts5HashEntry *pNext;
Fts5HashEntry *pSlot;
for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
pNext = pSlot->pHashNext;
sqlite3_free(pSlot);
}
}
memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
pHash->nEntry = 0;
}
static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
int i;
unsigned int h = 13;
for(i=n-1; i>=0; i--){
h = (h << 3) ^ h ^ p[i];
}
return (h % nSlot);
}
static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
int i;
unsigned int h = 13;
for(i=n-1; i>=0; i--){
h = (h << 3) ^ h ^ p[i];
}
h = (h << 3) ^ h ^ b;
return (h % nSlot);
}
/*
** Resize the hash table by doubling the number of slots.
*/
static int fts5HashResize(Fts5Hash *pHash){
int nNew = pHash->nSlot*2;
int i;
Fts5HashEntry **apNew;
Fts5HashEntry **apOld = pHash->aSlot;
apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
if( !apNew ) return SQLITE_NOMEM;
memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
for(i=0; i<pHash->nSlot; i++){
while( apOld[i] ){
unsigned int iHash;
Fts5HashEntry *p = apOld[i];
apOld[i] = p->pHashNext;
iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
(int)strlen(fts5EntryKey(p)));
p->pHashNext = apNew[iHash];
apNew[iHash] = p;
}
}
sqlite3_free(apOld);
pHash->nSlot = nNew;
pHash->aSlot = apNew;
return SQLITE_OK;
}
static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
if( p->iSzPoslist ){
u8 *pPtr = (u8*)p;
if( pHash->eDetail==FTS5_DETAIL_NONE ){
assert( p->nData==p->iSzPoslist );
if( p->bDel ){
pPtr[p->nData++] = 0x00;
if( p->bContent ){
pPtr[p->nData++] = 0x00;
}
}
}else{
int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
int nPos = nSz*2 + p->bDel; /* Value of nPos field */
assert( p->bDel==0 || p->bDel==1 );
if( nPos<=127 ){
pPtr[p->iSzPoslist] = (u8)nPos;
}else{
int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
p->nData += (nByte-1);
}
}
p->iSzPoslist = 0;
p->bDel = 0;
p->bContent = 0;
}
}
/*
** Add an entry to the in-memory hash table. The key is the concatenation
** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
**
** (bByte || pToken) -> (iRowid,iCol,iPos)
**
** Or, if iCol is negative, then the value is a delete marker.
*/
int sqlite3Fts5HashWrite(
Fts5Hash *pHash,
i64 iRowid, /* Rowid for this entry */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
char bByte, /* First byte of token */
const char *pToken, int nToken /* Token to add or remove to or from index */
){
unsigned int iHash;
Fts5HashEntry *p;
u8 *pPtr;
int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
int bNew; /* If non-delete entry should be written */
bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
/* Attempt to locate an existing hash entry */
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
char *zKey = fts5EntryKey(p);
if( zKey[0]==bByte
&& p->nKey==nToken
&& memcmp(&zKey[1], pToken, nToken)==0
){
break;
}
}
/* If an existing hash entry cannot be found, create a new one. */
if( p==0 ){
/* Figure out how much space to allocate */
char *zKey;
int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
if( nByte<128 ) nByte = 128;
/* Grow the Fts5Hash.aSlot[] array if necessary. */
if( (pHash->nEntry*2)>=pHash->nSlot ){
int rc = fts5HashResize(pHash);
if( rc!=SQLITE_OK ) return rc;
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
}
/* Allocate new Fts5HashEntry and add it to the hash table. */
p = (Fts5HashEntry*)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, sizeof(Fts5HashEntry));
p->nAlloc = nByte;
zKey = fts5EntryKey(p);
zKey[0] = bByte;
memcpy(&zKey[1], pToken, nToken);
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
p->nKey = nToken;
zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
/* Add the first rowid field to the hash-entry */
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
p->iRowid = iRowid;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
}
nIncr += p->nData;
}else{
/* Appending to an existing hash-entry. Check that there is enough
** space to append the largest possible new entry. Worst case scenario
** is:
**
** + 9 bytes for a new rowid,
** + 4 byte reserved for the "poslist size" varint.
** + 1 byte for a "new column" byte,
** + 3 bytes for a new column number (16-bit max) as a varint,
** + 5 bytes for the new position offset (32-bit max).
*/
if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
int nNew = p->nAlloc * 2;
Fts5HashEntry *pNew;
Fts5HashEntry **pp;
pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
if( pNew==0 ) return SQLITE_NOMEM;
pNew->nAlloc = nNew;
for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
*pp = pNew;
p = pNew;
}
nIncr -= p->nData;
}
assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
pPtr = (u8*)p;
/* If this is a new rowid, append the 4-byte size field for the previous
** entry, and the new rowid for this entry. */
if( iRowid!=p->iRowid ){
fts5HashAddPoslistSize(pHash, p);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
p->iRowid = iRowid;
bNew = 1;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
p->iPos = 0;
}
}
if( iCol>=0 ){
if( pHash->eDetail==FTS5_DETAIL_NONE ){
p->bContent = 1;
}else{
/* Append a new column value, if necessary */
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
if( pHash->eDetail==FTS5_DETAIL_FULL ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = (i16)iCol;
p->iPos = 0;
}else{
bNew = 1;
p->iCol = (i16)(iPos = iCol);
}
}
/* Append the new position offset, if necessary */
if( bNew ){
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->iPos = iPos;
}
}
}else{
/* This is a delete. Set the delete flag. */
p->bDel = 1;
}
nIncr += p->nData;
*pHash->pnByte += nIncr;
return SQLITE_OK;
}
/*
** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
** each sorted in key order. This function merges the two lists into a
** single list and returns a pointer to its first element.
*/
static Fts5HashEntry *fts5HashEntryMerge(
Fts5HashEntry *pLeft,
Fts5HashEntry *pRight
){
Fts5HashEntry *p1 = pLeft;
Fts5HashEntry *p2 = pRight;
Fts5HashEntry *pRet = 0;
Fts5HashEntry **ppOut = &pRet;
while( p1 || p2 ){
if( p1==0 ){
*ppOut = p2;
p2 = 0;
}else if( p2==0 ){
*ppOut = p1;
p1 = 0;
}else{
int i = 0;
char *zKey1 = fts5EntryKey(p1);
char *zKey2 = fts5EntryKey(p2);
while( zKey1[i]==zKey2[i] ) i++;
if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
/* p2 is smaller */
*ppOut = p2;
ppOut = &p2->pScanNext;
p2 = p2->pScanNext;
}else{
/* p1 is smaller */
*ppOut = p1;
ppOut = &p1->pScanNext;
p1 = p1->pScanNext;
}
*ppOut = 0;
}
}
return pRet;
}
/*
** Extract all tokens from hash table iHash and link them into a list
** in sorted order. The hash table is cleared before returning. It is
** the responsibility of the caller to free the elements of the returned
** list.
*/
static int fts5HashEntrySort(
Fts5Hash *pHash,
const char *pTerm, int nTerm, /* Query prefix, if any */
Fts5HashEntry **ppSorted
){
const int nMergeSlot = 32;
Fts5HashEntry **ap;
Fts5HashEntry *pList;
int iSlot;
int i;
*ppSorted = 0;
ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot);
if( !ap ) return SQLITE_NOMEM;
memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
Fts5HashEntry *pIter;
for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){
Fts5HashEntry *pEntry = pIter;
pEntry->pScanNext = 0;
for(i=0; ap[i]; i++){
pEntry = fts5HashEntryMerge(pEntry, ap[i]);
ap[i] = 0;
}
ap[i] = pEntry;
}
}
}
pList = 0;
for(i=0; i<nMergeSlot; i++){
pList = fts5HashEntryMerge(pList, ap[i]);
}
pHash->nEntry = 0;
sqlite3_free(ap);
*ppSorted = pList;
return SQLITE_OK;
}
/*
** Query the hash table for a doclist associated with term pTerm/nTerm.
*/
int sqlite3Fts5HashQuery(
Fts5Hash *pHash, /* Hash table to query */
const char *pTerm, int nTerm, /* Query term */
const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
int *pnDoclist /* OUT: Size of doclist in bytes */
){
unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
char *zKey = 0;
Fts5HashEntry *p;
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
zKey = fts5EntryKey(p);
if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break;
}
if( p ){
fts5HashAddPoslistSize(pHash, p);
*ppDoclist = (const u8*)&zKey[nTerm+1];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
}else{
*ppDoclist = 0;
*pnDoclist = 0;
}
return SQLITE_OK;
}
int sqlite3Fts5HashScanInit(
Fts5Hash *p, /* Hash table to query */
const char *pTerm, int nTerm /* Query prefix */
){
return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
}
void sqlite3Fts5HashScanNext(Fts5Hash *p){
assert( !sqlite3Fts5HashScanEof(p) );
p->pScan = p->pScan->pScanNext;
}
int sqlite3Fts5HashScanEof(Fts5Hash *p){
return (p->pScan==0);
}
void sqlite3Fts5HashScanEntry(
Fts5Hash *pHash,
const char **pzTerm, /* OUT: term (nul-terminated) */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
){
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
char *zKey = fts5EntryKey(p);
int nTerm = (int)strlen(zKey);
fts5HashAddPoslistSize(pHash, p);
*pzTerm = zKey;
*ppDoclist = (const u8*)&zKey[nTerm+1];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
}else{
*pzTerm = 0;
*ppDoclist = 0;
*pnDoclist = 0;
}
}
File diff suppressed because it is too large Load Diff
-2746
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-1154
View File
File diff suppressed because it is too large Load Diff
-421
View File
@@ -1,421 +0,0 @@
/*
** 2015 Aug 04
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains test code only, it is not included in release
** versions of FTS5. It contains the implementation of an FTS5 auxiliary
** function very similar to the FTS4 function matchinfo():
**
** https://www.sqlite.org/fts3.html#matchinfo
**
** Known differences are that:
**
** 1) this function uses the FTS5 definition of "matchable phrase", which
** excludes any phrases that are part of an expression sub-tree that
** does not match the current row. This comes up for MATCH queries
** such as:
**
** "a OR (b AND c)"
**
** In FTS4, if a single row contains instances of tokens "a" and "c",
** but not "b", all instances of "c" are considered matches. In FTS5,
** they are not (as the "b AND c" sub-tree does not match the current
** row.
**
** 2) For the values returned by 'x' that apply to all rows of the table,
** NEAR constraints are not considered. But for the number of hits in
** the current row, they are.
**
** This file exports a single function that may be called to register the
** matchinfo() implementation with a database handle:
**
** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
*/
#ifdef SQLITE_ENABLE_FTS5
#include "fts5.h"
#include <assert.h>
#include <string.h>
typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
#ifndef SQLITE_AMALGAMATION
typedef unsigned int u32;
#endif
struct Fts5MatchinfoCtx {
int nCol; /* Number of cols in FTS5 table */
int nPhrase; /* Number of phrases in FTS5 query */
char *zArg; /* nul-term'd copy of 2nd arg */
int nRet; /* Number of elements in aRet[] */
u32 *aRet; /* Array of 32-bit unsigned ints to return */
};
/*
** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
sqlite3_stmt *pStmt = 0;
int rc;
*ppApi = 0;
rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
(void)sqlite3_step(pStmt);
rc = sqlite3_finalize(pStmt);
}
return rc;
}
/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, -1 is
** returned. Otherwise, if f is a valid matchinfo flag, the value returned
** is the number of 32-bit integers added to the output array if the
** table has nCol columns and the query nPhrase phrases.
*/
static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
int ret = -1;
switch( f ){
case 'p': ret = 1; break;
case 'c': ret = 1; break;
case 'x': ret = 3 * nCol * nPhrase; break;
case 'y': ret = nCol * nPhrase; break;
case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
case 'n': ret = 1; break;
case 'a': ret = nCol; break;
case 'l': ret = nCol; break;
case 's': ret = nCol; break;
}
return ret;
}
static int fts5MatchinfoIter(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
Fts5MatchinfoCtx *p,
int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
){
int i;
int n = 0;
int rc = SQLITE_OK;
char f;
for(i=0; (f = p->zArg[i]); i++){
rc = x(pApi, pFts, p, f, &p->aRet[n]);
if( rc!=SQLITE_OK ) break;
n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
}
return rc;
}
static int fts5MatchinfoXCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
void *pUserData
){
Fts5PhraseIter iter;
int iCol, iOff;
u32 *aOut = (u32*)pUserData;
int iPrev = -1;
for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
iCol>=0;
pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
){
aOut[iCol*3+1]++;
if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
iPrev = iCol;
}
return SQLITE_OK;
}
static int fts5MatchinfoGlobalCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5MatchinfoCtx *p,
char f,
u32 *aOut
){
int rc = SQLITE_OK;
switch( f ){
case 'p':
aOut[0] = p->nPhrase;
break;
case 'c':
aOut[0] = p->nCol;
break;
case 'x': {
int i;
for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
void *pPtr = (void*)&aOut[i * p->nCol * 3];
rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
}
break;
}
case 'n': {
sqlite3_int64 nRow;
rc = pApi->xRowCount(pFts, &nRow);
aOut[0] = (u32)nRow;
break;
}
case 'a': {
sqlite3_int64 nRow = 0;
rc = pApi->xRowCount(pFts, &nRow);
if( nRow==0 ){
memset(aOut, 0, sizeof(u32) * p->nCol);
}else{
int i;
for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
sqlite3_int64 nToken;
rc = pApi->xColumnTotalSize(pFts, i, &nToken);
if( rc==SQLITE_OK){
aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
}
}
}
break;
}
}
return rc;
}
static int fts5MatchinfoLocalCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5MatchinfoCtx *p,
char f,
u32 *aOut
){
int i;
int rc = SQLITE_OK;
switch( f ){
case 'b': {
int iPhrase;
int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
for(i=0; i<nInt; i++) aOut[i] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;
int iCol;
for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
iCol>=0;
pApi->xPhraseNextColumn(pFts, &iter, &iCol)
){
aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
}
}
break;
}
case 'x':
case 'y': {
int nMul = (f=='x' ? 3 : 1);
int iPhrase;
for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;
int iOff, iCol;
for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
iOff>=0;
pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
){
aOut[nMul * (iCol + iPhrase * p->nCol)]++;
}
}
break;
}
case 'l': {
for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
int nToken;
rc = pApi->xColumnSize(pFts, i, &nToken);
aOut[i] = (u32)nToken;
}
break;
}
case 's': {
int nInst;
memset(aOut, 0, sizeof(u32) * p->nCol);
rc = pApi->xInstCount(pFts, &nInst);
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int iPhrase, iOff, iCol = 0;
int iNextPhrase;
int iNextOff;
u32 nSeq = 1;
int j;
rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
iNextPhrase = iPhrase+1;
iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
int ip, ic, io;
rc = pApi->xInst(pFts, j, &ip, &ic, &io);
if( ic!=iCol || io>iNextOff ) break;
if( ip==iNextPhrase && io==iNextOff ){
nSeq++;
iNextPhrase = ip+1;
iNextOff = io + pApi->xPhraseSize(pFts, ip);
}
}
if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
}
break;
}
}
return rc;
}
static Fts5MatchinfoCtx *fts5MatchinfoNew(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning error message */
const char *zArg /* Matchinfo flag string */
){
Fts5MatchinfoCtx *p;
int nCol;
int nPhrase;
int i;
int nInt;
int nByte;
int rc;
nCol = pApi->xColumnCount(pFts);
nPhrase = pApi->xPhraseCount(pFts);
nInt = 0;
for(i=0; zArg[i]; i++){
int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
if( n<0 ){
char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
return 0;
}
nInt += n;
}
nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
+ sizeof(u32) * nInt /* The p->aRet[] array */
+ (i+1); /* The p->zArg string */
p = (Fts5MatchinfoCtx*)sqlite3_malloc(nByte);
if( p==0 ){
sqlite3_result_error_nomem(pCtx);
return 0;
}
memset(p, 0, nByte);
p->nCol = nCol;
p->nPhrase = nPhrase;
p->aRet = (u32*)&p[1];
p->nRet = nInt;
p->zArg = (char*)&p->aRet[nInt];
memcpy(p->zArg, zArg, i);
rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(p);
p = 0;
}
return p;
}
static void fts5MatchinfoFunc(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
const char *zArg;
Fts5MatchinfoCtx *p;
int rc = SQLITE_OK;
if( nVal>0 ){
zArg = (const char*)sqlite3_value_text(apVal[0]);
}else{
zArg = "pcx";
}
p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
}
}
if( rc==SQLITE_OK ){
rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}else{
/* No errors has occured, so return a copy of the array of integers. */
int nByte = p->nRet * sizeof(u32);
sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
}
}
int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
int rc; /* Return code */
fts5_api *pApi; /* FTS5 API functions */
/* Extract the FTS5 API pointer from the database handle. The
** fts5_api_from_db() function above is copied verbatim from the
** FTS5 documentation. Refer there for details. */
rc = fts5_api_from_db(db, &pApi);
if( rc!=SQLITE_OK ) return rc;
/* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
** with this database handle, or an error (OOM perhaps?) has occurred.
**
** Also check that the fts5_api object is version 2 or newer.
*/
if( pApi==0 || pApi->iVersion<2 ){
return SQLITE_ERROR;
}
/* Register the implementation of matchinfo() */
rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
return rc;
}
#endif /* SQLITE_ENABLE_FTS5 */
-482
View File
@@ -1,482 +0,0 @@
/*
** 2013 Apr 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains code for the "fts5tokenize" virtual table module.
** An fts5tokenize virtual table is created as follows:
**
** CREATE VIRTUAL TABLE <tbl> USING fts5tokenize(
** <tokenizer-name>, <arg-1>, ...
** );
**
** The table created has the following schema:
**
** CREATE TABLE <tbl>(input HIDDEN, token, start, end, position)
**
** When queried, the query must include a WHERE clause of type:
**
** input = <string>
**
** The virtual table module tokenizes this <string>, using the FTS3
** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE
** statement and returns one row for each token in the result. With
** fields set as follows:
**
** input: Always set to a copy of <string>
** token: A token from the input.
** start: Byte offset of the token within the input <string>.
** end: Byte offset of the byte immediately following the end of the
** token within the input string.
** pos: Token offset of token within input.
**
*/
#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
#include "fts5.h"
#include <string.h>
#include <assert.h>
typedef struct Fts5tokTable Fts5tokTable;
typedef struct Fts5tokCursor Fts5tokCursor;
typedef struct Fts5tokRow Fts5tokRow;
/*
** Virtual table structure.
*/
struct Fts5tokTable {
sqlite3_vtab base; /* Base class used by SQLite core */
fts5_tokenizer tok; /* Tokenizer functions */
Fts5Tokenizer *pTok; /* Tokenizer instance */
};
/*
** A container for a rows values.
*/
struct Fts5tokRow {
char *zToken;
int iStart;
int iEnd;
int iPos;
};
/*
** Virtual table cursor structure.
*/
struct Fts5tokCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
int iRowid; /* Current 'rowid' value */
char *zInput; /* Input string */
int nRow; /* Number of entries in aRow[] */
Fts5tokRow *aRow; /* Array of rows to return */
};
static void fts5tokDequote(char *z){
char q = z[0];
if( q=='[' || q=='\'' || q=='"' || q=='`' ){
int iIn = 1;
int iOut = 0;
if( q=='[' ) q = ']';
while( z[iIn] ){
if( z[iIn]==q ){
if( z[iIn+1]!=q ){
/* Character iIn was the close quote. */
iIn++;
break;
}else{
/* Character iIn and iIn+1 form an escaped quote character. Skip
** the input cursor past both and copy a single quote character
** to the output buffer. */
iIn += 2;
z[iOut++] = q;
}
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
}
}
/*
** The second argument, argv[], is an array of pointers to nul-terminated
** strings. This function makes a copy of the array and strings into a
** single block of memory. It then dequotes any of the strings that appear
** to be quoted.
**
** If successful, output parameter *pazDequote is set to point at the
** array of dequoted strings and SQLITE_OK is returned. The caller is
** responsible for eventually calling sqlite3_free() to free the array
** in this case. Or, if an error occurs, an SQLite error code is returned.
** The final value of *pazDequote is undefined in this case.
*/
static int fts5tokDequoteArray(
int argc, /* Number of elements in argv[] */
const char * const *argv, /* Input array */
char ***pazDequote /* Output array */
){
int rc = SQLITE_OK; /* Return code */
if( argc==0 ){
*pazDequote = 0;
}else{
int i;
int nByte = 0;
char **azDequote;
for(i=0; i<argc; i++){
nByte += (int)(strlen(argv[i]) + 1);
}
*pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
if( azDequote==0 ){
rc = SQLITE_NOMEM;
}else{
char *pSpace = (char *)&azDequote[argc];
for(i=0; i<argc; i++){
int n = (int)strlen(argv[i]);
azDequote[i] = pSpace;
memcpy(pSpace, argv[i], n+1);
fts5tokDequote(pSpace);
pSpace += (n+1);
}
}
}
return rc;
}
/*
** Schema of the tokenizer table.
*/
#define FTS3_TOK_SCHEMA "CREATE TABLE x(input HIDDEN, token, start, end, position)"
/*
** This function does all the work for both the xConnect and xCreate methods.
** These tables have no persistent representation of their own, so xConnect
** and xCreate are identical operations.
**
** argv[0]: module name
** argv[1]: database name
** argv[2]: table name
** argv[3]: first argument (tokenizer name)
*/
static int fts5tokConnectMethod(
sqlite3 *db, /* Database connection */
void *pCtx, /* Pointer to fts5_api object */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
fts5_api *pApi = (fts5_api*)pCtx;
Fts5tokTable *pTab = 0;
int rc;
char **azDequote = 0;
int nDequote = 0;
rc = sqlite3_declare_vtab(db,
"CREATE TABLE x(input HIDDEN, token, start, end, position)"
);
if( rc==SQLITE_OK ){
nDequote = argc-3;
rc = fts5tokDequoteArray(nDequote, &argv[3], &azDequote);
}
if( rc==SQLITE_OK ){
pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable));
if( pTab==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pTab, 0, sizeof(Fts5tokTable));
}
}
if( rc==SQLITE_OK ){
void *pTokCtx = 0;
const char *zModule = 0;
if( nDequote>0 ){
zModule = azDequote[0];
}
rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);
if( rc==SQLITE_OK ){
const char **azArg = (const char **)&azDequote[1];
int nArg = nDequote>0 ? nDequote-1 : 0;
rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(pTab);
pTab = 0;
}
*ppVtab = (sqlite3_vtab*)pTab;
sqlite3_free(azDequote);
return rc;
}
/*
** This function does the work for both the xDisconnect and xDestroy methods.
** These tables have no persistent representation of their own, so xDisconnect
** and xDestroy are identical operations.
*/
static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){
Fts5tokTable *pTab = (Fts5tokTable *)pVtab;
if( pTab->pTok ){
pTab->tok.xDelete(pTab->pTok);
}
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts5tokBestIndexMethod(
sqlite3_vtab *pVTab,
sqlite3_index_info *pInfo
){
int i;
for(i=0; i<pInfo->nConstraint; i++){
if( pInfo->aConstraint[i].usable
&& pInfo->aConstraint[i].iColumn==0
&& pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ
){
pInfo->idxNum = 1;
pInfo->aConstraintUsage[i].argvIndex = 1;
pInfo->aConstraintUsage[i].omit = 1;
pInfo->estimatedCost = 1;
return SQLITE_OK;
}
}
pInfo->idxNum = 0;
assert( pInfo->estimatedCost>1000000.0 );
return SQLITE_OK;
}
/*
** xOpen - Open a cursor.
*/
static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5tokCursor *pCsr;
pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));
if( pCsr==0 ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(Fts5tokCursor));
*ppCsr = (sqlite3_vtab_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Reset the tokenizer cursor passed as the only argument. As if it had
** just been returned by fts5tokOpenMethod().
*/
static void fts5tokResetCursor(Fts5tokCursor *pCsr){
int i;
for(i=0; i<pCsr->nRow; i++){
sqlite3_free(pCsr->aRow[i].zToken);
}
sqlite3_free(pCsr->zInput);
sqlite3_free(pCsr->aRow);
pCsr->zInput = 0;
pCsr->aRow = 0;
pCsr->nRow = 0;
pCsr->iRowid = 0;
}
/*
** xClose - Close a cursor.
*/
static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
fts5tokResetCursor(pCsr);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** xNext - Advance the cursor to the next row, if any.
*/
static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
pCsr->iRowid++;
return SQLITE_OK;
}
static int fts5tokCb(
void *pCtx, /* Pointer to Fts5tokCursor */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
){
Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;
Fts5tokRow *pRow;
if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
Fts5tokRow *aNew;
aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
if( aNew==0 ) return SQLITE_NOMEM;
memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
pCsr->aRow = aNew;
}
pRow = &pCsr->aRow[pCsr->nRow];
pRow->iStart = iStart;
pRow->iEnd = iEnd;
if( pCsr->nRow ){
pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);
}
pRow->zToken = sqlite3_malloc(nToken+1);
if( pRow->zToken==0 ) return SQLITE_NOMEM;
memcpy(pRow->zToken, pToken, nToken);
pRow->zToken[nToken] = 0;
pCsr->nRow++;
return SQLITE_OK;
}
/*
** xFilter - Initialize a cursor to point at the start of its data.
*/
static int fts5tokFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
int rc = SQLITE_ERROR;
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);
fts5tokResetCursor(pCsr);
if( idxNum==1 ){
const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
int nByte = sqlite3_value_bytes(apVal[0]);
pCsr->zInput = sqlite3_malloc(nByte+1);
if( pCsr->zInput==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zInput, zByte, nByte);
pCsr->zInput[nByte] = 0;
rc = pTab->tok.xTokenize(
pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
);
}
}
if( rc!=SQLITE_OK ) return rc;
return fts5tokNextMethod(pCursor);
}
/*
** xEof - Return true if the cursor is at EOF, or false otherwise.
*/
static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
return (pCsr->iRowid>pCsr->nRow);
}
/*
** xColumn - Return a column value.
*/
static int fts5tokColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];
/* CREATE TABLE x(input, token, start, end, position) */
switch( iCol ){
case 0:
sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
break;
case 1:
sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);
break;
case 2:
sqlite3_result_int(pCtx, pRow->iStart);
break;
case 3:
sqlite3_result_int(pCtx, pRow->iEnd);
break;
default:
assert( iCol==4 );
sqlite3_result_int(pCtx, pRow->iPos);
break;
}
return SQLITE_OK;
}
/*
** xRowid - Return the current rowid for the cursor.
*/
static int fts5tokRowidMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite_int64 *pRowid /* OUT: Rowid value */
){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
*pRowid = (sqlite3_int64)pCsr->iRowid;
return SQLITE_OK;
}
/*
** Register the fts5tok module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
static const sqlite3_module fts5tok_module = {
0, /* iVersion */
fts5tokConnectMethod, /* xCreate */
fts5tokConnectMethod, /* xConnect */
fts5tokBestIndexMethod, /* xBestIndex */
fts5tokDisconnectMethod, /* xDisconnect */
fts5tokDisconnectMethod, /* xDestroy */
fts5tokOpenMethod, /* xOpen */
fts5tokCloseMethod, /* xClose */
fts5tokFilterMethod, /* xFilter */
fts5tokNextMethod, /* xNext */
fts5tokEofMethod, /* xEof */
fts5tokColumnMethod, /* xColumn */
fts5tokRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0 /* xRollbackTo */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
return rc;
}
#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */
File diff suppressed because it is too large Load Diff
-360
View File
@@ -1,360 +0,0 @@
/*
** 2012 May 25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
#include <assert.h>
/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeIsalnum(int c){
/* Each unsigned integer in the following array corresponds to a contiguous
** range of unicode codepoints that are not either letters or numbers (i.e.
** codepoints for which this function should return 0).
**
** The most significant 22 bits in each 32-bit value contain the first
** codepoint in the range. The least significant 10 bits are used to store
** the size of the range (always at least 1). In other words, the value
** ((C<<22) + N) represents a range of N codepoints starting with codepoint
** C. It is not possible to represent a range larger than 1023 codepoints
** using this format.
*/
static const unsigned int aEntry[] = {
0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
0x380400F0,
};
static const unsigned int aAscii[4] = {
0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
};
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aEntry[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( aEntry[0]<key );
assert( key>=aEntry[iRes] );
return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
}
return 1;
}
/*
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int fts5_remove_diacritic(int c){
unsigned short aDia[] = {
0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
62924, 63050, 63082, 63274, 63390,
};
char aChar[] = {
'\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
'e', 'i', 'o', 'u', 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
int iRes = 0;
int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aDia[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
int sqlite3Fts5UnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** codepoints starting at codepoint iCode.
**
** If the least significant bit in flags is clear, then the rule applies
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
{386, 1, 4}, {390, 44, 1}, {391, 0, 1},
{393, 42, 2}, {395, 0, 1}, {398, 32, 1},
{399, 38, 1}, {400, 40, 1}, {401, 0, 1},
{403, 42, 1}, {404, 46, 1}, {406, 52, 1},
{407, 48, 1}, {408, 0, 1}, {412, 52, 1},
{413, 54, 1}, {415, 56, 1}, {416, 1, 6},
{422, 60, 1}, {423, 0, 1}, {425, 60, 1},
{428, 0, 1}, {430, 60, 1}, {431, 0, 1},
{433, 58, 2}, {435, 1, 4}, {439, 62, 1},
{440, 0, 1}, {444, 0, 1}, {452, 2, 1},
{453, 0, 1}, {455, 2, 1}, {456, 0, 1},
{458, 2, 1}, {459, 1, 18}, {478, 1, 18},
{497, 2, 1}, {498, 1, 4}, {502, 122, 1},
{503, 134, 1}, {504, 1, 40}, {544, 110, 1},
{546, 1, 18}, {570, 70, 1}, {571, 0, 1},
{573, 108, 1}, {574, 68, 1}, {577, 0, 1},
{579, 106, 1}, {580, 28, 1}, {581, 30, 1},
{582, 1, 10}, {837, 36, 1}, {880, 1, 4},
{886, 0, 1}, {902, 18, 1}, {904, 16, 3},
{908, 26, 1}, {910, 24, 2}, {913, 14, 17},
{931, 14, 9}, {962, 0, 1}, {975, 4, 1},
{976, 140, 1}, {977, 142, 1}, {981, 146, 1},
{982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
{1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
{1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
{1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
{1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
{1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
{4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
{7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
{7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
{7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
{8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
{8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
{8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
{8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
{8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
{8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
{8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
{8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
{8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
{11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
{11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
{11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
{11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
{11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
{42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
{42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
{42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
{42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
{65313, 14, 26},
};
static const unsigned short aiOff[] = {
1, 2, 8, 15, 16, 26, 28, 32,
37, 38, 40, 48, 63, 64, 69, 71,
79, 80, 116, 202, 203, 205, 206, 207,
209, 210, 211, 213, 214, 217, 218, 219,
775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
65514, 65521, 65527, 65528, 65529,
};
int ret = c;
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
if( cmp>=0 ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
}
else if( c>=66560 && c<66600 ){
ret = c + 40;
}
return ret;
}
-345
View File
@@ -1,345 +0,0 @@
/*
** 2015 May 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Routines for varint serialization and deserialization.
*/
#include "fts5Int.h"
/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
u32 a,b;
/* The 1-byte case. Overwhelmingly the most common. */
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
/* Values between 0 and 127 */
*v = a;
return 1;
}
/* The 2-byte case */
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
/* Values between 128 and 16383 */
a &= 0x7f;
a = a<<7;
*v = a | b;
return 2;
}
/* The 3-byte case */
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
/* Values between 16384 and 2097151 */
a &= (0x7f<<14)|(0x7f);
b &= 0x7f;
b = b<<7;
*v = a | b;
return 3;
}
/* A 32-bit varint is used to store size information in btrees.
** Objects are rarely larger than 2MiB limit of a 3-byte varint.
** A 3-byte varint is sufficient, for example, to record the size
** of a 1048569-byte BLOB or string.
**
** We only unroll the first 1-, 2-, and 3- byte cases. The very
** rare larger cases can be handled by the slower 64-bit varint
** routine.
*/
{
u64 v64;
u8 n;
p -= 2;
n = sqlite3Fts5GetVarint(p, &v64);
*v = (u32)v64;
assert( n>3 && n<=9 );
return n;
}
}
/*
** Bitmasks used by sqlite3GetVarint(). These precomputed constants
** are defined here rather than simply putting the constant expressions
** inline in order to work around bugs in the RVT compiler.
**
** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
**
** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
*/
#define SLOT_2_0 0x001fc07f
#define SLOT_4_2_0 0xf01fc07f
/*
** Read a 64-bit variable-length integer from memory starting at p[0].
** Return the number of bytes read. The value is stored in *v.
*/
u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
u32 a,b,s;
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
*v = a;
return 1;
}
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
a &= 0x7f;
a = a<<7;
a |= b;
*v = a;
return 2;
}
/* Verify that constants are precomputed correctly */
assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) );
assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) );
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_2_0;
b &= 0x7f;
b = b<<7;
a |= b;
*v = a;
return 3;
}
/* CSE1 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p1<<14 | p3 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_2_0;
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
*v = a;
return 4;
}
/* a: p0<<14 | p2 (masked) */
/* b: p1<<14 | p3 (unmasked) */
/* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
b &= SLOT_2_0;
s = a;
/* s: p0<<14 | p2 (masked) */
p++;
a = a<<14;
a |= *p;
/* a: p0<<28 | p2<<14 | p4 (unmasked) */
if (!(a&0x80))
{
/* we can skip these cause they were (effectively) done above in calc'ing s */
/* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
/* b &= (0x7f<<14)|(0x7f); */
b = b<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 5;
}
/* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
s = s<<7;
s |= b;
/* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
p++;
b = b<<14;
b |= *p;
/* b: p1<<28 | p3<<14 | p5 (unmasked) */
if (!(b&0x80))
{
/* we can skip this cause it was (effectively) done above in calc'ing s */
/* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
a &= SLOT_2_0;
a = a<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 6;
}
p++;
a = a<<14;
a |= *p;
/* a: p2<<28 | p4<<14 | p6 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_4_2_0;
b &= SLOT_2_0;
b = b<<7;
a |= b;
s = s>>11;
*v = ((u64)s)<<32 | a;
return 7;
}
/* CSE2 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p3<<28 | p5<<14 | p7 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_4_2_0;
/* moved CSE2 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
s = s>>4;
*v = ((u64)s)<<32 | a;
return 8;
}
p++;
a = a<<15;
a |= *p;
/* a: p4<<29 | p6<<15 | p8 (unmasked) */
/* moved CSE2 up */
/* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
b &= SLOT_2_0;
b = b<<8;
a |= b;
s = s<<4;
b = p[-4];
b &= 0x7f;
b = b>>3;
s |= b;
*v = ((u64)s)<<32 | a;
return 9;
}
/*
** The variable-length integer encoding is as follows:
**
** KEY:
** A = 0xxxxxxx 7 bits of data and one flag bit
** B = 1xxxxxxx 7 bits of data and one flag bit
** C = xxxxxxxx 8 bits of data
**
** 7 bits - A
** 14 bits - BA
** 21 bits - BBA
** 28 bits - BBBA
** 35 bits - BBBBA
** 42 bits - BBBBBA
** 49 bits - BBBBBBA
** 56 bits - BBBBBBBA
** 64 bits - BBBBBBBBC
*/
#ifdef SQLITE_NOINLINE
# define FTS5_NOINLINE SQLITE_NOINLINE
#else
# define FTS5_NOINLINE
#endif
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data write will be between 1 and 9 bytes. The number
** of bytes written is returned.
**
** A variable-length integer consists of the lower 7 bits of each byte
** for all bytes that have the 8th bit set and one byte with the 8th
** bit clear. Except, if we get to the 9th byte, it stores the full
** 8 bits and is the last byte.
*/
static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
int i, j, n;
u8 buf[10];
if( v & (((u64)0xff000000)<<32) ){
p[8] = (u8)v;
v >>= 8;
for(i=7; i>=0; i--){
p[i] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}
return 9;
}
n = 0;
do{
buf[n++] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}while( v!=0 );
buf[0] &= 0x7f;
assert( n<=9 );
for(i=0, j=n-1; j>=0; j--, i++){
p[i] = buf[j];
}
return n;
}
int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
if( v<=0x7f ){
p[0] = v&0x7f;
return 1;
}
if( v<=0x3fff ){
p[0] = ((v>>7)&0x7f)|0x80;
p[1] = v&0x7f;
return 2;
}
return fts5PutVarint64(p,v);
}
int sqlite3Fts5GetVarintLen(u32 iVal){
#if 0
if( iVal<(1 << 7 ) ) return 1;
#endif
assert( iVal>=(1 << 7) );
if( iVal<(1 << 14) ) return 2;
if( iVal<(1 << 21) ) return 3;
if( iVal<(1 << 28) ) return 4;
return 5;
}
-764
View File
@@ -1,764 +0,0 @@
/*
** 2015 May 08
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite virtual table module implementing direct access to an
** existing FTS5 index. The module may create several different types of
** tables:
**
** col:
** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
**
** One row for each term/column combination. The value of $doc is set to
** the number of fts5 rows that contain at least one instance of term
** $term within column $col. Field $cnt is set to the total number of
** instances of term $term in column $col (in any row of the fts5 table).
**
** row:
** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
**
** One row for each term in the database. The value of $doc is set to
** the number of fts5 rows that contain at least one instance of term
** $term. Field $cnt is set to the total number of instances of term
** $term in the database.
**
** instance:
** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
**
** One row for each term instance in the database.
*/
#include "fts5Int.h"
typedef struct Fts5VocabTable Fts5VocabTable;
typedef struct Fts5VocabCursor Fts5VocabCursor;
struct Fts5VocabTable {
sqlite3_vtab base;
char *zFts5Tbl; /* Name of fts5 table */
char *zFts5Db; /* Db containing fts5 table */
sqlite3 *db; /* Database handle */
Fts5Global *pGlobal; /* FTS5 global object for this database */
int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */
};
struct Fts5VocabCursor {
sqlite3_vtab_cursor base;
sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
Fts5Index *pIndex; /* Associated FTS5 index */
int bEof; /* True if this cursor is at EOF */
Fts5IndexIter *pIter; /* Term/rowid iterator object */
int nLeTerm; /* Size of zLeTerm in bytes */
char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
/* These are used by 'col' tables only */
Fts5Config *pConfig; /* Fts5 table configuration */
int iCol;
i64 *aCnt;
i64 *aDoc;
/* Output values used by all tables. */
i64 rowid; /* This table's current rowid value */
Fts5Buffer term; /* Current value of 'term' column */
/* Output values Used by 'instance' tables only */
i64 iInstPos;
int iInstOff;
};
#define FTS5_VOCAB_COL 0
#define FTS5_VOCAB_ROW 1
#define FTS5_VOCAB_INSTANCE 2
#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt"
#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt"
#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"
/*
** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
*/
#define FTS5_VOCAB_TERM_EQ 0x01
#define FTS5_VOCAB_TERM_GE 0x02
#define FTS5_VOCAB_TERM_LE 0x04
/*
** Translate a string containing an fts5vocab table type to an
** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
** and return SQLITE_ERROR.
*/
static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
int rc = SQLITE_OK;
char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
if( rc==SQLITE_OK ){
sqlite3Fts5Dequote(zCopy);
if( sqlite3_stricmp(zCopy, "col")==0 ){
*peType = FTS5_VOCAB_COL;
}else
if( sqlite3_stricmp(zCopy, "row")==0 ){
*peType = FTS5_VOCAB_ROW;
}else
if( sqlite3_stricmp(zCopy, "instance")==0 ){
*peType = FTS5_VOCAB_INSTANCE;
}else
{
*pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
rc = SQLITE_ERROR;
}
sqlite3_free(zCopy);
}
return rc;
}
/*
** The xDisconnect() virtual table method.
*/
static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** The xDestroy() virtual table method.
*/
static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
** argv[0] -> module name ("fts5vocab")
** argv[1] -> database name
** argv[2] -> table name
**
** then:
**
** argv[3] -> name of fts5 table
** argv[4] -> type of fts5vocab table
**
** or, for tables in the TEMP schema only.
**
** argv[3] -> name of fts5 tables database
** argv[4] -> name of fts5 table
** argv[5] -> type of fts5vocab table
*/
static int fts5VocabInitVtab(
sqlite3 *db, /* The SQLite database connection */
void *pAux, /* Pointer to Fts5Global object */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
const char *azSchema[] = {
"CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")",
"CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")",
"CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
};
Fts5VocabTable *pRet = 0;
int rc = SQLITE_OK; /* Return code */
int bDb;
bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
if( argc!=5 && bDb==0 ){
*pzErr = sqlite3_mprintf("wrong number of vtable arguments");
rc = SQLITE_ERROR;
}else{
int nByte; /* Bytes of space to allocate */
const char *zDb = bDb ? argv[3] : argv[1];
const char *zTab = bDb ? argv[4] : argv[3];
const char *zType = bDb ? argv[5] : argv[4];
int nDb = (int)strlen(zDb)+1;
int nTab = (int)strlen(zTab)+1;
int eType = 0;
rc = fts5VocabTableType(zType, pzErr, &eType);
if( rc==SQLITE_OK ){
assert( eType>=0 && eType<ArraySize(azSchema) );
rc = sqlite3_declare_vtab(db, azSchema[eType]);
}
nByte = sizeof(Fts5VocabTable) + nDb + nTab;
pRet = sqlite3Fts5MallocZero(&rc, nByte);
if( pRet ){
pRet->pGlobal = (Fts5Global*)pAux;
pRet->eType = eType;
pRet->db = db;
pRet->zFts5Tbl = (char*)&pRet[1];
pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
memcpy(pRet->zFts5Tbl, zTab, nTab);
memcpy(pRet->zFts5Db, zDb, nDb);
sqlite3Fts5Dequote(pRet->zFts5Tbl);
sqlite3Fts5Dequote(pRet->zFts5Db);
}
}
*ppVTab = (sqlite3_vtab*)pRet;
return rc;
}
/*
** The xConnect() and xCreate() methods for the virtual table. All the
** work is done in function fts5VocabInitVtab().
*/
static int fts5VocabConnectMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
}
static int fts5VocabCreateMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
}
/*
** Implementation of the xBestIndex method.
**
** Only constraints of the form:
**
** term <= ?
** term == ?
** term >= ?
**
** are interpreted. Less-than and less-than-or-equal are treated
** identically, as are greater-than and greater-than-or-equal.
*/
static int fts5VocabBestIndexMethod(
sqlite3_vtab *pUnused,
sqlite3_index_info *pInfo
){
int i;
int iTermEq = -1;
int iTermGe = -1;
int iTermLe = -1;
int idxNum = 0;
int nArg = 0;
UNUSED_PARAM(pUnused);
for(i=0; i<pInfo->nConstraint; i++){
struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
if( p->usable==0 ) continue;
if( p->iColumn==0 ){ /* term column */
if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
}
}
if( iTermEq>=0 ){
idxNum |= FTS5_VOCAB_TERM_EQ;
pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
pInfo->estimatedCost = 100;
}else{
pInfo->estimatedCost = 1000000;
if( iTermGe>=0 ){
idxNum |= FTS5_VOCAB_TERM_GE;
pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
pInfo->estimatedCost = pInfo->estimatedCost / 2;
}
if( iTermLe>=0 ){
idxNum |= FTS5_VOCAB_TERM_LE;
pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
pInfo->estimatedCost = pInfo->estimatedCost / 2;
}
}
/* This virtual table always delivers results in ascending order of
** the "term" column (column 0). So if the user has requested this
** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
** sqlite3_index_info.orderByConsumed flag to tell the core the results
** are already in sorted order. */
if( pInfo->nOrderBy==1
&& pInfo->aOrderBy[0].iColumn==0
&& pInfo->aOrderBy[0].desc==0
){
pInfo->orderByConsumed = 1;
}
pInfo->idxNum = idxNum;
return SQLITE_OK;
}
/*
** Implementation of xOpen method.
*/
static int fts5VocabOpenMethod(
sqlite3_vtab *pVTab,
sqlite3_vtab_cursor **ppCsr
){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
Fts5Index *pIndex = 0;
Fts5Config *pConfig = 0;
Fts5VocabCursor *pCsr = 0;
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
char *zSql = 0;
zSql = sqlite3Fts5Mprintf(&rc,
"SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
);
if( zSql ){
rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
}
sqlite3_free(zSql);
assert( rc==SQLITE_OK || pStmt==0 );
if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
i64 iId = sqlite3_column_int64(pStmt, 0);
pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
}
if( rc==SQLITE_OK && pIndex==0 ){
rc = sqlite3_finalize(pStmt);
pStmt = 0;
if( rc==SQLITE_OK ){
pVTab->zErrMsg = sqlite3_mprintf(
"no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
);
rc = SQLITE_ERROR;
}
}
if( rc==SQLITE_OK ){
int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
}
if( pCsr ){
pCsr->pIndex = pIndex;
pCsr->pStmt = pStmt;
pCsr->pConfig = pConfig;
pCsr->aCnt = (i64*)&pCsr[1];
pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
}else{
sqlite3_finalize(pStmt);
}
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
}
static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
pCsr->rowid = 0;
sqlite3Fts5IterClose(pCsr->pIter);
pCsr->pIter = 0;
sqlite3_free(pCsr->zLeTerm);
pCsr->nLeTerm = -1;
pCsr->zLeTerm = 0;
}
/*
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
fts5VocabResetCursor(pCsr);
sqlite3Fts5BufferFree(&pCsr->term);
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr);
return SQLITE_OK;
}
static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
int rc = SQLITE_OK;
if( sqlite3Fts5IterEof(pCsr->pIter) ){
pCsr->bEof = 1;
}else{
const char *zTerm;
int nTerm;
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( pCsr->nLeTerm>=0 ){
int nCmp = MIN(nTerm, pCsr->nLeTerm);
int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
pCsr->bEof = 1;
}
}
sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
}
return rc;
}
static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
int eDetail = pCsr->pConfig->eDetail;
int rc = SQLITE_OK;
Fts5IndexIter *pIter = pCsr->pIter;
i64 *pp = &pCsr->iInstPos;
int *po = &pCsr->iInstOff;
while( eDetail==FTS5_DETAIL_NONE
|| sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp)
){
pCsr->iInstPos = 0;
pCsr->iInstOff = 0;
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
if( rc==SQLITE_OK ){
rc = fts5VocabInstanceNewTerm(pCsr);
if( eDetail==FTS5_DETAIL_NONE ) break;
}
if( rc ){
pCsr->bEof = 1;
break;
}
}
return rc;
}
/*
** Advance the cursor to the next row in the table.
*/
static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
int rc = SQLITE_OK;
int nCol = pCsr->pConfig->nCol;
pCsr->rowid++;
if( pTab->eType==FTS5_VOCAB_INSTANCE ){
return fts5VocabInstanceNext(pCsr);
}
if( pTab->eType==FTS5_VOCAB_COL ){
for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
if( pCsr->aDoc[pCsr->iCol] ) break;
}
}
if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
if( sqlite3Fts5IterEof(pCsr->pIter) ){
pCsr->bEof = 1;
}else{
const char *zTerm;
int nTerm;
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( pCsr->nLeTerm>=0 ){
int nCmp = MIN(nTerm, pCsr->nLeTerm);
int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
pCsr->bEof = 1;
return SQLITE_OK;
}
}
sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
memset(pCsr->aCnt, 0, nCol * sizeof(i64));
memset(pCsr->aDoc, 0, nCol * sizeof(i64));
pCsr->iCol = 0;
assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
while( rc==SQLITE_OK ){
int eDetail = pCsr->pConfig->eDetail;
const u8 *pPos; int nPos; /* Position list */
i64 iPos = 0; /* 64-bit position read from poslist */
int iOff = 0; /* Current offset within position list */
pPos = pCsr->pIter->pData;
nPos = pCsr->pIter->nData;
switch( pTab->eType ){
case FTS5_VOCAB_ROW:
if( eDetail==FTS5_DETAIL_FULL ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
}
}
pCsr->aDoc[0]++;
break;
case FTS5_VOCAB_COL:
if( eDetail==FTS5_DETAIL_FULL ){
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
if( ii>=nCol ){
rc = FTS5_CORRUPT;
break;
}
pCsr->aDoc[ii]++;
iCol = ii;
}
}
}else if( eDetail==FTS5_DETAIL_COLUMNS ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
assert_nc( iPos>=0 && iPos<nCol );
if( iPos>=nCol ){
rc = FTS5_CORRUPT;
break;
}
pCsr->aDoc[iPos]++;
}
}else{
assert( eDetail==FTS5_DETAIL_NONE );
pCsr->aDoc[0]++;
}
break;
default:
assert( pTab->eType==FTS5_VOCAB_INSTANCE );
break;
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
}
if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;
if( rc==SQLITE_OK ){
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
break;
}
if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
}
}
}
if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
assert( pCsr->iCol<pCsr->pConfig->nCol );
}
return rc;
}
/*
** This is the xFilter implementation for the virtual table.
*/
static int fts5VocabFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *zUnused, /* Unused */
int nUnused, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
int eType = pTab->eType;
int rc = SQLITE_OK;
int iVal = 0;
int f = FTS5INDEX_QUERY_SCAN;
const char *zTerm = 0;
int nTerm = 0;
sqlite3_value *pEq = 0;
sqlite3_value *pGe = 0;
sqlite3_value *pLe = 0;
UNUSED_PARAM2(zUnused, nUnused);
fts5VocabResetCursor(pCsr);
if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
if( pEq ){
zTerm = (const char *)sqlite3_value_text(pEq);
nTerm = sqlite3_value_bytes(pEq);
f = 0;
}else{
if( pGe ){
zTerm = (const char *)sqlite3_value_text(pGe);
nTerm = sqlite3_value_bytes(pGe);
}
if( pLe ){
const char *zCopy = (const char *)sqlite3_value_text(pLe);
pCsr->nLeTerm = sqlite3_value_bytes(pLe);
pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
if( pCsr->zLeTerm==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
}
if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
rc = fts5VocabInstanceNewTerm(pCsr);
}
if( rc==SQLITE_OK
&& !pCsr->bEof
&& (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE)
){
rc = fts5VocabNextMethod(pCursor);
}
return rc;
}
/*
** This is the xEof method of the virtual table. SQLite calls this
** routine to find out if it has reached the end of a result set.
*/
static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
return pCsr->bEof;
}
static int fts5VocabColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
int eDetail = pCsr->pConfig->eDetail;
int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
i64 iVal = 0;
if( iCol==0 ){
sqlite3_result_text(
pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
);
}else if( eType==FTS5_VOCAB_COL ){
assert( iCol==1 || iCol==2 || iCol==3 );
if( iCol==1 ){
if( eDetail!=FTS5_DETAIL_NONE ){
const char *z = pCsr->pConfig->azCol[pCsr->iCol];
sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
}
}else if( iCol==2 ){
iVal = pCsr->aDoc[pCsr->iCol];
}else{
iVal = pCsr->aCnt[pCsr->iCol];
}
}else if( eType==FTS5_VOCAB_ROW ){
assert( iCol==1 || iCol==2 );
if( iCol==1 ){
iVal = pCsr->aDoc[0];
}else{
iVal = pCsr->aCnt[0];
}
}else{
assert( eType==FTS5_VOCAB_INSTANCE );
switch( iCol ){
case 1:
sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
break;
case 2: {
int ii = -1;
if( eDetail==FTS5_DETAIL_FULL ){
ii = FTS5_POS2COLUMN(pCsr->iInstPos);
}else if( eDetail==FTS5_DETAIL_COLUMNS ){
ii = (int)pCsr->iInstPos;
}
if( ii>=0 && ii<pCsr->pConfig->nCol ){
const char *z = pCsr->pConfig->azCol[ii];
sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
}
break;
}
default: {
assert( iCol==3 );
if( eDetail==FTS5_DETAIL_FULL ){
int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
sqlite3_result_int(pCtx, ii);
}
break;
}
}
}
if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
return SQLITE_OK;
}
/*
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. The
** rowid should be written to *pRowid.
*/
static int fts5VocabRowidMethod(
sqlite3_vtab_cursor *pCursor,
sqlite_int64 *pRowid
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
*pRowid = pCsr->rowid;
return SQLITE_OK;
}
int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
static const sqlite3_module fts5Vocab = {
/* iVersion */ 2,
/* xCreate */ fts5VocabCreateMethod,
/* xConnect */ fts5VocabConnectMethod,
/* xBestIndex */ fts5VocabBestIndexMethod,
/* xDisconnect */ fts5VocabDisconnectMethod,
/* xDestroy */ fts5VocabDestroyMethod,
/* xOpen */ fts5VocabOpenMethod,
/* xClose */ fts5VocabCloseMethod,
/* xFilter */ fts5VocabFilterMethod,
/* xNext */ fts5VocabNextMethod,
/* xEof */ fts5VocabEofMethod,
/* xColumn */ fts5VocabColumnMethod,
/* xRowid */ fts5VocabRowidMethod,
/* xUpdate */ 0,
/* xBegin */ 0,
/* xSync */ 0,
/* xCommit */ 0,
/* xRollback */ 0,
/* xFindFunction */ 0,
/* xRename */ 0,
/* xSavepoint */ 0,
/* xRelease */ 0,
/* xRollbackTo */ 0,
};
void *p = (void*)pGlobal;
return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
}
-197
View File
@@ -1,197 +0,0 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
// All token codes are small integers with #defines that begin with "TK_"
%token_prefix FTS5_
// The type of the data attached to each token is Token. This is also the
// default type for non-terminals.
//
%token_type {Fts5Token}
%default_type {Fts5Token}
// The generated parser function takes a 4th argument as follows:
%extra_argument {Fts5Parse *pParse}
// This code runs whenever there is a syntax error
//
%syntax_error {
UNUSED_PARAM(yymajor); /* Silence a compiler warning */
sqlite3Fts5ParseError(
pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p
);
}
%stack_overflow {
sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
}
// The name of the generated procedure that implements the parser
// is as follows:
%name sqlite3Fts5Parser
// The following text is included near the beginning of the C source
// code file that implements the parser.
//
%include {
#include "fts5Int.h"
#include "fts5parse.h"
/*
** Disable all error recovery processing in the parser push-down
** automaton.
*/
#define YYNOERRORRECOVERY 1
/*
** Make yytestcase() the same as testcase()
*/
#define yytestcase(X) testcase(X)
/*
** Indicate that sqlite3ParserFree() will never be called with a null
** pointer.
*/
#define YYPARSEFREENOTNULL 1
/*
** Alternative datatype for the argument to the malloc() routine passed
** into sqlite3ParserAlloc(). The default is size_t.
*/
#define YYMALLOCARGTYPE u64
} // end %include
%left OR.
%left AND.
%left NOT.
%left TERM.
%left COLON.
input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
%destructor input { (void)pParse; }
%type cnearset {Fts5ExprNode*}
%type expr {Fts5ExprNode*}
%type exprlist {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= MINUS LCP colsetlist(X) RCP. {
A = sqlite3Fts5ParseColsetInvert(pParse, X);
}
colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colset(A) ::= MINUS STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
A = sqlite3Fts5ParseColsetInvert(pParse, A);
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
expr(A) ::= expr(X) AND expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}
expr(A) ::= colset(X) COLON LP expr(Y) RP. {
sqlite3Fts5ParseSetColset(pParse, Y, X);
A = Y;
}
expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X). {A = X;}
exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}
cnearset(A) ::= nearset(X). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
}
cnearset(A) ::= colset(X) COLON nearset(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
sqlite3Fts5ParseSetColset(pParse, A, X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
nearset(A) ::= CARET phrase(Y). {
sqlite3Fts5ParseSetCaret(Y);
A = sqlite3Fts5ParseNearset(pParse, 0, Y);
}
nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
sqlite3Fts5ParseNear(pParse, &X);
sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
A = Y;
}
nearphrases(A) ::= phrase(X). {
A = sqlite3Fts5ParseNearset(pParse, 0, X);
}
nearphrases(A) ::= nearphrases(X) phrase(Y). {
A = sqlite3Fts5ParseNearset(pParse, X, Y);
}
/*
** The optional ", <integer>" at the end of the NEAR() arguments.
*/
neardist_opt(A) ::= . { A.p = 0; A.n = 0; }
neardist_opt(A) ::= COMMA STRING(X). { A = X; }
/*
** A phrase. A set of primitives connected by "+" operators. Examples:
**
** "the" + "quick brown" + fo *
** "the quick brown fo" *
** the+quick+brown+fo*
*/
%type phrase {Fts5ExprPhrase*}
%destructor phrase { sqlite3Fts5ParsePhraseFree($$); }
phrase(A) ::= phrase(X) PLUS STRING(Y) star_opt(Z). {
A = sqlite3Fts5ParseTerm(pParse, X, &Y, Z);
}
phrase(A) ::= STRING(Y) star_opt(Z). {
A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z);
}
/*
** Optional "*" character.
*/
%type star_opt {int}
star_opt(A) ::= STAR. { A = 1; }
star_opt(A) ::= . { A = 0; }
-222
View File
@@ -1,222 +0,0 @@
#
# 2014 Jun 09
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#-------------------------------------------------------------------------
#
# This script generates the implementations of the following C functions,
# which are part of the porter tokenizer implementation:
#
# static int fts5PorterStep1B(char *aBuf, int *pnBuf);
# static int fts5PorterStep1B2(char *aBuf, int *pnBuf);
# static int fts5PorterStep2(char *aBuf, int *pnBuf);
# static int fts5PorterStep3(char *aBuf, int *pnBuf);
# static int fts5PorterStep4(char *aBuf, int *pnBuf);
#
set O(Step1B2) {
{ at {} ate 1 }
{ bl {} ble 1 }
{ iz {} ize 1 }
}
set O(Step1B) {
{ "eed" fts5Porter_MGt0 "ee" 0 }
{ "ed" fts5Porter_Vowel "" 1 }
{ "ing" fts5Porter_Vowel "" 1 }
}
set O(Step2) {
{ "ational" fts5Porter_MGt0 "ate" }
{ "tional" fts5Porter_MGt0 "tion" }
{ "enci" fts5Porter_MGt0 "ence" }
{ "anci" fts5Porter_MGt0 "ance" }
{ "izer" fts5Porter_MGt0 "ize" }
{ "logi" fts5Porter_MGt0 "log" }
{ "bli" fts5Porter_MGt0 "ble" }
{ "alli" fts5Porter_MGt0 "al" }
{ "entli" fts5Porter_MGt0 "ent" }
{ "eli" fts5Porter_MGt0 "e" }
{ "ousli" fts5Porter_MGt0 "ous" }
{ "ization" fts5Porter_MGt0 "ize" }
{ "ation" fts5Porter_MGt0 "ate" }
{ "ator" fts5Porter_MGt0 "ate" }
{ "alism" fts5Porter_MGt0 "al" }
{ "iveness" fts5Porter_MGt0 "ive" }
{ "fulness" fts5Porter_MGt0 "ful" }
{ "ousness" fts5Porter_MGt0 "ous" }
{ "aliti" fts5Porter_MGt0 "al" }
{ "iviti" fts5Porter_MGt0 "ive" }
{ "biliti" fts5Porter_MGt0 "ble" }
}
set O(Step3) {
{ "icate" fts5Porter_MGt0 "ic" }
{ "ative" fts5Porter_MGt0 "" }
{ "alize" fts5Porter_MGt0 "al" }
{ "iciti" fts5Porter_MGt0 "ic" }
{ "ical" fts5Porter_MGt0 "ic" }
{ "ful" fts5Porter_MGt0 "" }
{ "ness" fts5Porter_MGt0 "" }
}
set O(Step4) {
{ "al" fts5Porter_MGt1 "" }
{ "ance" fts5Porter_MGt1 "" }
{ "ence" fts5Porter_MGt1 "" }
{ "er" fts5Porter_MGt1 "" }
{ "ic" fts5Porter_MGt1 "" }
{ "able" fts5Porter_MGt1 "" }
{ "ible" fts5Porter_MGt1 "" }
{ "ant" fts5Porter_MGt1 "" }
{ "ement" fts5Porter_MGt1 "" }
{ "ment" fts5Porter_MGt1 "" }
{ "ent" fts5Porter_MGt1 "" }
{ "ion" fts5Porter_MGt1_and_S_or_T "" }
{ "ou" fts5Porter_MGt1 "" }
{ "ism" fts5Porter_MGt1 "" }
{ "ate" fts5Porter_MGt1 "" }
{ "iti" fts5Porter_MGt1 "" }
{ "ous" fts5Porter_MGt1 "" }
{ "ive" fts5Porter_MGt1 "" }
{ "ize" fts5Porter_MGt1 "" }
}
proc sort_cb {lhs rhs} {
set L [string range [lindex $lhs 0] end-1 end-1]
set R [string range [lindex $rhs 0] end-1 end-1]
string compare $L $R
}
proc create_step_function {name data} {
set T(function) {
static int fts5Porter${name}(char *aBuf, int *pnBuf){
int ret = 0;
int nBuf = *pnBuf;
switch( aBuf[nBuf-2] ){
${switchbody}
}
return ret;
}
}
set T(case) {
case '${k}':
${ifstmts}
break;
}
set T(if_0_0_0) {
if( ${match} ){
*pnBuf = nBuf - $n;
}
}
set T(if_1_0_0) {
if( ${match} ){
if( ${cond} ){
*pnBuf = nBuf - $n;
}
}
}
set T(if_0_1_0) {
if( ${match} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
}
}
set T(if_1_1_0) {
if( ${match} ){
if( ${cond} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
}
}
}
set T(if_1_0_1) {
if( ${match} ){
if( ${cond} ){
*pnBuf = nBuf - $n;
ret = 1;
}
}
}
set T(if_0_1_1) {
if( ${match} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
ret = 1;
}
}
set T(if_1_1_1) {
if( ${match} ){
if( ${cond} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
ret = 1;
}
}
}
set switchbody ""
foreach I $data {
set k [string range [lindex $I 0] end-1 end-1]
lappend aCase($k) $I
}
foreach k [lsort [array names aCase]] {
set ifstmts ""
foreach I $aCase($k) {
set zSuffix [lindex $I 0] ;# Suffix text for this rule
set zRep [lindex $I 2] ;# Replacement text for rule
set xCond [lindex $I 1] ;# Condition callback (or "")
set n [string length $zSuffix]
set nRep [string length $zRep]
set match "nBuf>$n && 0==memcmp(\"$zSuffix\", &aBuf\[nBuf-$n\], $n)"
set memcpy "memcpy(&aBuf\[nBuf-$n\], \"$zRep\", $nRep);"
set cond "${xCond}(aBuf, nBuf-$n)"
set bMemcpy [expr {$nRep>0}]
set bCond [expr {$xCond!=""}]
set bRet [expr {[llength $I]>3 && [lindex $I 3]}]
set t $T(if_${bCond}_${bMemcpy}_${bRet})
lappend ifstmts [string trim [subst -nocommands $t]]
}
set ifstmts [join $ifstmts "else "]
append switchbody [subst -nocommands $T(case)]
}
puts [subst -nocommands $T(function)]
}
puts [string trim {
/**************************************************************************
***************************************************************************
** GENERATED CODE STARTS HERE (mkportersteps.tcl)
*/
}]
foreach step [array names O] {
create_step_function $step $O($step)
}
puts [string trim {
/*
** GENERATED CODE ENDS HERE (mkportersteps.tcl)
***************************************************************************
**************************************************************************/
}]
-647
View File
@@ -1,647 +0,0 @@
# 2014 Dec 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. .. test]
}
source $testdir/tester.tcl
ifcapable !fts5 {
proc return_if_no_fts5 {} {
finish_test
return -code return
}
return
} else {
proc return_if_no_fts5 {} {}
}
catch {
sqlite3_fts5_may_be_corrupt 0
reset_db
}
proc fts5_test_poslist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
lappend res [string map {{ } .} [$cmd xInst $i]]
}
set res
}
proc fts5_test_poslist2 {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseForeach $i c o {
lappend res $i.$c.$o
}
}
#set res
sort_poslist $res
}
proc fts5_test_collist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseColumnForeach $i c { lappend res $i.$c }
}
set res
}
proc fts5_test_columnsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnSize $i]
}
set res
}
proc fts5_test_columntext {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnText $i]
}
set res
}
proc fts5_test_columntotalsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnTotalSize $i]
}
set res
}
proc test_append_token {varname token iStart iEnd} {
upvar $varname var
lappend var $token
return "SQLITE_OK"
}
proc fts5_test_tokenize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
set tokens [list]
$cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
lappend res $tokens
}
set res
}
proc fts5_test_rowcount {cmd} {
$cmd xRowCount
}
proc test_queryphrase_cb {cnt cmd} {
upvar $cnt L
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
foreach {ip ic io} [$cmd xInst $i] break
set A($ic) 1
}
foreach ic [array names A] {
lset L $ic [expr {[lindex $L $ic] + 1}]
}
}
proc fts5_test_queryphrase {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
set cnt [list]
for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
$cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
lappend res $cnt
}
set res
}
proc fts5_test_phrasecount {cmd} {
$cmd xPhraseCount
}
proc fts5_test_all {cmd} {
set res [list]
lappend res columnsize [fts5_test_columnsize $cmd]
lappend res columntext [fts5_test_columntext $cmd]
lappend res columntotalsize [fts5_test_columntotalsize $cmd]
lappend res poslist [fts5_test_poslist $cmd]
lappend res tokenize [fts5_test_tokenize $cmd]
lappend res rowcount [fts5_test_rowcount $cmd]
set res
}
proc fts5_aux_test_functions {db} {
foreach f {
fts5_test_columnsize
fts5_test_columntext
fts5_test_columntotalsize
fts5_test_poslist
fts5_test_poslist2
fts5_test_collist
fts5_test_tokenize
fts5_test_rowcount
fts5_test_all
fts5_test_queryphrase
fts5_test_phrasecount
} {
sqlite3_fts5_create_function $db $f $f
}
}
proc fts5_segcount {tbl} {
set N 0
foreach n [fts5_level_segs $tbl] { incr N $n }
set N
}
proc fts5_level_segs {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
lappend ret [expr [llength $L] - 3]
}
set ret
}
proc fts5_level_segids {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
set lvl [list]
foreach S [lrange $L 3 end] {
regexp {id=([1234567890]*)} $S -> segid
lappend lvl $segid
}
lappend ret $lvl
}
set ret
}
proc fts5_rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
}
set doc
}
#-------------------------------------------------------------------------
# Usage:
#
# nearset aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
# This command is used to test if a document (set of column values) matches
# the logical equivalent of a single FTS5 NEAR() clump and, if so, return
# the equivalent of an FTS5 position list.
#
# Parameter $aCol is passed a list of the column values for the document
# to test. Parameters $phrase1 and so on are the phrases.
#
# The result is a list of phrase hits. Each phrase hit is formatted as
# three integers separated by "." characters, in the following format:
#
# <phrase number> . <column number> . <token offset>
#
# Options:
#
# -near N (NEAR distance. Default 10)
# -col C (List of column indexes to match against)
# -pc VARNAME (variable in caller frame to use for phrase numbering)
# -dict VARNAME (array in caller frame to use for synonyms)
#
proc nearset {aCol args} {
# Process the command line options.
#
set O(-near) 10
set O(-col) {}
set O(-pc) ""
set O(-dict) ""
set nOpt [lsearch -exact $args --]
if {$nOpt<0} { error "no -- option" }
# Set $lPhrase to be a list of phrases. $nPhrase its length.
set lPhrase [lrange $args [expr $nOpt+1] end]
set nPhrase [llength $lPhrase]
foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
if {[info exists O($k)]==0} { error "unrecognized option $k" }
set O($k) $v
}
if {$O(-pc) == ""} {
set counter 0
} else {
upvar $O(-pc) counter
}
if {$O(-dict)!=""} { upvar $O(-dict) aDict }
for {set j 0} {$j < [llength $aCol]} {incr j} {
for {set i 0} {$i < $nPhrase} {incr i} {
set A($j,$i) [list]
}
}
# Loop through each column of the current row.
for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
# If there is a column filter, test whether this column is excluded. If
# so, skip to the next iteration of this loop. Otherwise, set zCol to the
# column value and nToken to the number of tokens that comprise it.
if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
set zCol [lindex $aCol $iCol]
set nToken [llength $zCol]
# Each iteration of the following loop searches a substring of the
# column value for phrase matches. The last token of the substring
# is token $iLast of the column value. The first token is:
#
# iFirst = ($iLast - $O(-near) - 1)
#
# where $sz is the length of the phrase being searched for. A phrase
# counts as matching the substring if its first token lies on or before
# $iLast and its last token on or after $iFirst.
#
# For example, if the query is "NEAR(a+b c, 2)" and the column value:
#
# "x x x x A B x x C x"
# 0 1 2 3 4 5 6 7 8 9"
#
# when (iLast==8 && iFirst=5) the range will contain both phrases and
# so both instances can be added to the output poslists.
#
set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
for { } {$iLast < $nToken} {incr iLast} {
catch { array unset B }
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set p [lindex $lPhrase $iPhrase]
set nPm1 [expr {[llength $p] - 1}]
set iFirst [expr $iLast - $O(-near) - [llength $p]]
for {set i $iFirst} {$i <= $iLast} {incr i} {
set lCand [lrange $zCol $i [expr $i+$nPm1]]
set bMatch 1
foreach tok $p term $lCand {
if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
}
if {$bMatch} { lappend B($iPhrase) $i }
}
if {![info exists B($iPhrase)]} break
}
if {$iPhrase==$nPhrase} {
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
}
}
}
}
set res [list]
#puts [array names A]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
foreach a $A($iCol,$iPhrase) {
lappend res "$counter.$iCol.$a"
}
}
incr counter
}
#puts "$aCol -> $res"
sort_poslist $res
}
proc nearset_match {aDictVar tok term} {
if {[string match $tok $term]} { return 1 }
upvar $aDictVar aDict
if {[info exists aDict($tok)]} {
foreach s $aDict($tok) {
if {[string match $s $term]} { return 1 }
}
}
return 0;
}
#-------------------------------------------------------------------------
# Usage:
#
# sort_poslist LIST
#
# Sort a position list of the type returned by command [nearset]
#
proc sort_poslist {L} {
lsort -command instcompare $L
}
proc instcompare {lhs rhs} {
foreach {p1 c1 o1} [split $lhs .] {}
foreach {p2 c2 o2} [split $rhs .] {}
set res [expr $c1 - $c2]
if {$res==0} { set res [expr $o1 - $o2] }
if {$res==0} { set res [expr $p1 - $p2] }
return $res
}
#-------------------------------------------------------------------------
# Logical operators used by the commands returned by fts5_tcl_expr().
#
proc AND {args} {
foreach a $args {
if {[llength $a]==0} { return [list] }
}
sort_poslist [concat {*}$args]
}
proc OR {args} {
sort_poslist [concat {*}$args]
}
proc NOT {a b} {
if {[llength $b]>0} { return [list] }
return $a
}
#-------------------------------------------------------------------------
# This command is similar to [split], except that it also provides the
# start and end offsets of each token. For example:
#
# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8}
#
proc gobble_whitespace {textvar} {
upvar $textvar t
regexp {([ ]*)(.*)} $t -> space t
return [string length $space]
}
proc gobble_text {textvar wordvar} {
upvar $textvar t
upvar $wordvar w
regexp {([^ ]*)(.*)} $t -> w t
return [string length $w]
}
proc fts5_tokenize_split {text} {
set token ""
set ret [list]
set iOff [gobble_whitespace text]
while {[set nToken [gobble_text text word]]} {
lappend ret $word $iOff [expr $iOff+$nToken]
incr iOff $nToken
incr iOff [gobble_whitespace text]
}
set ret
}
#-------------------------------------------------------------------------
#
proc foreach_detail_mode {prefix script} {
set saved $::testprefix
foreach d [list full col none] {
set s [string map [list %DETAIL% $d] $script]
set ::detail $d
set ::testprefix "$prefix-$d"
reset_db
uplevel $s
unset ::detail
}
set ::testprefix $saved
}
proc detail_check {} {
if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
error "not in foreach_detail_mode {...} block"
}
}
proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
#-------------------------------------------------------------------------
# Convert a poslist of the type returned by fts5_test_poslist() to a
# collist as returned by fts5_test_collist().
#
proc fts5_poslist2collist {poslist} {
set res [list]
foreach h $poslist {
regexp {(.*)\.[1234567890]+} $h -> cand
lappend res $cand
}
set res [lsort -command fts5_collist_elem_compare -unique $res]
return $res
}
# Comparison function used by fts5_poslist2collist to sort collist entries.
proc fts5_collist_elem_compare {a b} {
foreach {a1 a2} [split $a .] {}
foreach {b1 b2} [split $b .] {}
if {$a1==$b1} { return [expr $a2 - $b2] }
return [expr $a1 - $b1]
}
#--------------------------------------------------------------------------
# Construct and return a tcl list equivalent to that returned by the SQL
# query executed against database handle [db]:
#
# SELECT
# rowid,
# fts5_test_poslist($tbl),
# fts5_test_collist($tbl)
# FROM $tbl('$expr')
# ORDER BY rowid $order;
#
proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
# Figure out the set of columns in the FTS5 table. This routine does
# not handle tables with UNINDEXED columns, but if it did, it would
# have to be here.
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set d ""
if {$aDictVar != ""} {
upvar $aDictVar aDict
set d aDict
}
set cols ""
foreach e $lCols { append cols ", '$e'" }
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
}]]
set res [list]
db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
set cols [list]
foreach col $lCols { lappend cols $x($col) }
set ::pc 0
set rowdata [eval $tclexpr]
if {$rowdata != ""} {
lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
}
}
set res
}
#-------------------------------------------------------------------------
# Similar to [fts5_query_data], but omit the collist field.
#
proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
set res [list]
if {$aDictVar!=""} {
upvar $aDictVar aDict
set dict aDict
} else {
set dict ""
}
foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
lappend res $rowid $poslist
}
set res
}
proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
set res [list]
if {$aDictVar!=""} {
upvar $aDictVar aDict
set dict aDict
} else {
set dict ""
}
foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
lappend res $rowid $collist
}
set res
}
#-------------------------------------------------------------------------
#
# This command will only work inside a [foreach_detail_mode] block. It tests
# whether or not expression $expr run on FTS5 table $tbl is supported by
# the current mode. If so, 1 is returned. If not, 0.
#
# detail=full (all queries supported)
# detail=col (all but phrase queries and NEAR queries)
# detail=none (all but phrase queries, NEAR queries, and column filters)
#
proc fts5_expr_ok {expr tbl} {
if {![detail_is_full]} {
set nearset "nearset_rc"
if {[detail_is_col]} { set nearset "nearset_rf" }
set ::expr_not_ok 0
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set cols ""
foreach e $lCols { append cols ", '$e'" }
set ::pc 0
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
}]]
eval $tclexpr
if {$::expr_not_ok} { return 0 }
}
return 1
}
# Helper for [fts5_expr_ok]
proc nearset_rf {aCol args} {
set idx [lsearch -exact $args --]
if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
set ::expr_not_ok 1
}
list
}
# Helper for [fts5_expr_ok]
proc nearset_rc {aCol args} {
nearset_rf $aCol {*}$args
if {[lsearch $args -col]>=0} {
set ::expr_not_ok 1
}
list
}
#-------------------------------------------------------------------------
# Code for a simple Tcl tokenizer that supports synonyms at query time.
#
proc tclnum_tokenize {mode tflags text} {
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
}
}
}
proc tclnum_create {args} {
set mode query
if {[llength $args]} {
set mode [lindex $args 0]
}
if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
return [list tclnum_tokenize $mode]
}
proc fts5_tclnum_register {db} {
foreach SYNDICT {
{zero 0}
{one 1 i}
{two 2 ii}
{three 3 iii}
{four 4 iv}
{five 5 v}
{six 6 vi}
{seven 7 vii}
{eight 8 viii}
{nine 9 ix}
{a1 a2 a3 a4 a5 a6 a7 a8 a9}
{b1 b2 b3 b4 b5 b6 b7 b8 b9}
{c1 c2 c3 c4 c5 c6 c7 c8 c9}
} {
foreach s $SYNDICT {
set o [list]
foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
set ::tclnum_syn($s) $o
}
}
sqlite3_fts5_create_tokenizer db tclnum tclnum_create
}
#
# End of tokenizer code.
#-------------------------------------------------------------------------
-597
View File
@@ -1,597 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aa
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
} {
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID}
t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
}
do_execsql_test 1.1 {
DROP TABLE t1;
SELECT name, sql FROM sqlite_master;
} {
}
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_test 2.2 {
execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
foreach w {a b c d e f} {
do_execsql_test 2.3.$w.asc {
SELECT rowid FROM t1 WHERE t1 MATCH $w;
} {1}
do_execsql_test 2.3.$w.desc {
SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;
} {1}
}
do_execsql_test 2.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 4.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 4.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
1 {dd abc abc abc abcde} {aaa dd ddd ddd aab}
2 {dd aab d aaa b} {abcde c aaa aaa aaa}
3 {abcde dd b b dd} {abc abc d abc ddddd}
4 {aaa abcde dddd dddd abcde} {abc b b abcde abc}
5 {aab dddd d dddd c} {ddd abcde dddd abcde c}
6 {ddd dd b aab abcde} {d ddddd dddd c abc}
7 {d ddddd ddd c abcde} {c aab d abcde ddd}
8 {abcde aaa aab c c} {ddd c dddd b aaa}
9 {abcde aab ddddd c aab} {dddd dddd b c dd}
10 {ddd abcde dddd dd c} {dddd c c d abcde}
} {
do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 6.1 {
INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
}
do_execsql_test 6.2 {
INSERT INTO t1(t1) VALUES('integrity-check')
}
do_execsql_test 6.3 {
REPLACE INTO t1(rowid, x, y) VALUES('22', 'l l l', 'l l l');
}
do_execsql_test 6.4 {
REPLACE INTO t1(x, y) VALUES('x y z', 'x y z');
}
do_execsql_test 6.5 {
INSERT INTO t1(t1) VALUES('integrity-check')
}
do_execsql_test 6.6 {
SELECT rowid, * FROM t1;
} {
22 {l l l} {l l l}
23 {x y z} {x y z}
}
#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 7.$i {
for {set j 0} {$j < 10} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 8.1 {
INSERT INTO t1 VALUES('the quick brown fox');
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 9.$i {
for {set j 0} {$j < 100} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
set d10 {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
}
foreach {rowid x y} $d10 {
do_execsql_test 10.1.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 10.1.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach rowid {5 9 8 1 2 4 10 7 3 5 6} {
do_execsql_test 10.2.$rowid.1 { DELETE FROM t1 WHERE rowid = $rowid }
do_execsql_test 10.2.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach {rowid x y} $d10 {
do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
do_execsql_test 10.4.1 { DELETE FROM t1 }
do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
} {1 {reserved fts5 column name: rowid}}
#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
} {}
do_catchsql_test 12.2 {
SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}
do_test 12.3 {
set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
string is integer $res
} {1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
} {}
do_execsql_test 13.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1 2}
do_execsql_test 13.4 {
DELETE FROM t1 WHERE rowid=2;
} {}
do_execsql_test 13.5 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1}
do_execsql_test 13.6 {
SELECT rowid FROM t1 WHERE t1 MATCH '""';
} {}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
UNION ALL
SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d
)
INSERT INTO t1 SELECT * FROM d LIMIT 200;
}
do_execsql_test 15.x {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_test 14.2 {
set nRow 0
db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
db eval {
BEGIN;
CREATE TABLE t2(a, b);
ROLLBACK;
}
incr nRow
}
set nRow
} {200}
do_test 14.3 {
set nRow 0
db eval { BEGIN; }
db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
db eval {
SAVEPOINT aaa;
CREATE TABLE t2(a, b);
ROLLBACK TO aaa;
RELEASE aaa;
}
incr nRow
}
set nRow
} {200}
do_execsql_test 15.0 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_execsql_test 15.1 {
UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
}
do_catchsql_test 15.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
#
do_execsql_test 16.1 {
CREATE VIRTUAL TABLE n1 USING fts5(a);
INSERT INTO n1 VALUES('a b c d');
}
proc funk {} {
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
set fd [db incrblob main n1_data block 10]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd "\x44\x45"
close $fd
}
db func funk funk
# This test case corrupts the structure record within the first invocation
# of function funk(). Which used to cause the bm25() function to throw an
# exception. But since bm25() can now used the cached structure record,
# it never sees the corruption introduced by funk() and so the following
# statement no longer fails.
#
do_catchsql_test 16.2 {
SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {0 {{} -1e-06 {}}}
# {1 {SQL logic error}}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
INSERT INTO b2 VALUES('a');
INSERT INTO b2 VALUES('b');
INSERT INTO b2 VALUES('c');
}
do_test 17.2 {
set res [list]
db eval { SELECT * FROM b2 ORDER BY rowid ASC } {
lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }]
}
set res
} {{a b c} {a b c} {a b c}}
if {[string match n* %DETAIL%]==0} {
reset_db
do_execsql_test 17.3 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
INSERT INTO uio VALUES(NULL);
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
SELECT count(*) FROM uio;
} {256}
do_execsql_test 17.2 {
SELECT count(*) FROM uio WHERE rowid BETWEEN 8 AND 17
} {10}
do_execsql_test 17.3 {
SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17
} {8 9 10 11 12 13 14 15 16 17}
do_execsql_test 17.4 {
SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17 ORDER BY rowid DESC
} {17 16 15 14 13 12 11 10 9 8}
do_execsql_test 17.5 {
SELECT count(*) FROM uio
} {256}
do_execsql_test 17.6 {
INSERT INTO uio(rowid) VALUES(9223372036854775807);
INSERT INTO uio(rowid) VALUES(-9223372036854775808);
SELECT count(*) FROM uio;
} {258}
do_execsql_test 17.7 {
SELECT min(rowid), max(rowid) FROM uio;
} {-9223372036854775808 9223372036854775807}
do_execsql_test 17.8 {
INSERT INTO uio DEFAULT VALUES;
SELECT min(rowid), max(rowid), count(*) FROM uio;
} {-9223372036854775808 9223372036854775807 259}
do_execsql_test 17.9 {
SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10;
} {-9223372036854775808 9 10}
#--------------------------------------------------------------------
#
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
INSERT INTO t1 VALUES('abc*', NULL);
INSERT INTO t2 VALUES(1, 'abcdefg');
}
do_execsql_test 18.2 {
SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
} {1 1}
do_execsql_test 18.3 {
SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
} {1 1}
#--------------------------------------------------------------------
# fts5 table in the temp schema.
#
reset_db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('w x 1');
SELECT rowid FROM t1 WHERE t1 MATCH 'x';
} {1 2}
#--------------------------------------------------------------------
# Test that 6 and 7 byte varints can be read.
#
reset_db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
}
set ::ids [list \
0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
]
do_test 20.1 {
foreach id $::ids {
execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') }
}
execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
} $::ids
#--------------------------------------------------------------------
# Test that a DROP TABLE may be executed within a transaction that
# writes to an FTS5 table.
#
do_execsql_test 21.0 {
CREATE TEMP TABLE t8(a, b);
CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
}
do_execsql_test 21.1 {
BEGIN;
INSERT INTO ft VALUES('a b c');
DROP TABLE t8;
COMMIT;
}
do_execsql_test 22.0 {
CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
BEGIN;
INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
DELETE FROM t9 WHERE rowid = 2;
INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
COMMIT;
}
do_execsql_test 22.1 {
SELECT rowid FROM t9('a*')
} {1}
}
finish_test
-296
View File
@@ -1,296 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ab
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1 VALUES('hello', 'world');
INSERT INTO t1 VALUES('one two', 'three four');
INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
}
do_execsql_test 1.1 {
SELECT * FROM t1 ORDER BY rowid DESC;
} { forty five {one two} {three four} hello world }
do_execsql_test 1.2 {
SELECT rowid FROM t1 ORDER BY rowid DESC;
} {45 2 1}
do_execsql_test 1.3 {
SELECT rowid FROM t1 ORDER BY rowid ASC;
} {1 2 45}
do_execsql_test 1.4 {
SELECT * FROM t1 WHERE rowid=2;
} {{one two} {three four}}
do_execsql_test 1.5 {
SELECT * FROM t1 WHERE rowid=2.01;
} {}
do_execsql_test 1.6 {
SELECT * FROM t1 WHERE rowid=1.99;
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('one');
INSERT INTO t1 VALUES('two');
INSERT INTO t1 VALUES('three');
}
do_catchsql_test 2.2 {
SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
} {1 {fts5: syntax error near "AND"}}
do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two}
do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three}
do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one}
do_execsql_test 2.6 {
INSERT INTO t1 VALUES('a b c d e f g');
INSERT INTO t1 VALUES('b d e a a a i');
INSERT INTO t1 VALUES('x y z b c c c');
}
foreach {tn expr res} {
1 a {5 4}
2 b {6 5 4}
3 c {6 4}
4 d {5 4}
5 e {5 4}
6 f {4}
7 g {4}
8 x {6}
9 y {6}
10 z {6}
} {
do_execsql_test 2.7.$tn.1 {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC
} $res
do_execsql_test 2.7.$tn.2 {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC
} [lsort -integer $res]
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a,b);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {tn a b} {
1 {abashed abandons abase abash abaft} {abases abased}
2 {abasing abases abaft abated abandons} {abases abandoned}
3 {abatement abash abash abated abase} {abasements abashing}
4 {abaft abasements abase abasement abasing} {abasement abases}
5 {abaft abashing abatement abash abasements} {abandons abandoning}
6 {aback abate abasements abashes abandoned} {abasement abased}
7 {abandons abated abased aback abandoning} {abases abandoned}
8 {abashing abases abasement abaft abashing} {abashed abate}
9 {abash abase abate abashing abashed} {abandon abandoned}
10 {abate abandoning abandons abasement aback} {abandon abandoning}
} {
do_execsql_test 3.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) }
do_execsql_test 3.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach {tn expr res} {
1 {abash} {9 5 3 1}
2 {abase} {9 4 3 1}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {8 5}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 3.2.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC
} $res
}
do_execsql_test 3.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)'
} {6}
foreach {tn expr res} {
1 {abash} {1 3 5 9}
2 {abase} {1 3 4 9}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {5 8}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 3.4.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr
} $res
}
#-------------------------------------------------------------------------
# Documents with more than 2M tokens.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%);
}
foreach {tn doc} [list \
1 [string repeat {a x } 1500000] \
2 "[string repeat {a a } 1500000] x" \
] {
do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) }
}
do_execsql_test 4.3 {
SELECT rowid FROM s1 WHERE s1 MATCH 'x'
} {1 2}
if {[detail_is_full]} {
do_execsql_test 4.4 {
SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
} {1 2}
}
do_execsql_test 4.5 {
SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
} {1 2}
#-------------------------------------------------------------------------
# Check that a special case of segment promotion works. The case is where
# a new segment is written to level L, but the oldest segment within level
# (L-2) is larger than it.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
proc rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]]
}
set doc
}
db func rnddoc rnddoc
do_test 5.1 {
for {set i 1} {$i <= 65} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(10)) }
}
for {set i 1} {$i <= 63} {incr i} {
execsql { DELETE FROM s2 WHERE rowid = $i }
}
fts5_level_segs s2
} {0 8}
do_test 5.2 {
execsql {
INSERT INTO s2(s2, rank) VALUES('automerge', 8);
}
for {set i 0} {$i < 7} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(50)) }
}
fts5_level_segs s2
} {8 0 0}
# Test also the other type of segment promotion - when a new segment is written
# that is larger than segments immediately following it.
do_test 5.3 {
execsql {
DROP TABLE s2;
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
for {set i 1} {$i <= 16} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
}
fts5_level_segs s2
} {0 1}
do_test 5.4 {
execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
fts5_level_segs s2
} {2 0}
#-------------------------------------------------------------------------
#
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%);
BEGIN;
INSERT INTO s3 VALUES('a b c');
INSERT INTO s3 VALUES('A B C');
}
do_execsql_test 6.1.1 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a'
} {1 2}
do_execsql_test 6.1.2 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a' ORDER BY rowid DESC
} {2 1}
do_execsql_test 6.2 {
COMMIT;
}
do_execsql_test 6.3 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a'
} {1 2}
do_test 6.4 {
db close
sqlite3 db test.db
execsql {
BEGIN;
INSERT INTO s3(s3) VALUES('optimize');
ROLLBACK;
}
} {}
#-------------------------------------------------------------------------
#
set doc [string repeat "a b c " 500]
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
INSERT INTO x1 VALUES($doc);
}
} ;# foreach_detail_mode...
finish_test
-278
View File
@@ -1,278 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ac
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
set data {
0 {p o q e z k z p n f y u z y n y} {l o o l v v k}
1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w}
2 {l s z j k i m p s} {l w e j t j e e i t w r o p o}
3 {x g y m y m h p} {k j j b r e y y a k y}
4 {q m a i y i z} {o w a g k x g j m w e u k}
5 {k o a w y b s z} {s g l m m l m g p}
6 {d a q i z h b l c p k j g k} {p x u j x t v c z}
7 {f d a g o c t i} {w f c x l d r k i j}
8 {y g w u b q p o m j y b p a e k} {r i d k y w o z q m a t p}
9 {r k o m c c j s x m x m x m q r} {y r c a q d z k n x n}
10 {k j q m g q a j d} {d d e z g w h c d o o g x d}
11 {j z u m o y q j f w e e w t r j w} {g m o r x n t n w i f g l z f}
12 {s y w a w d o h x m k} {c w k z b p o r a}
13 {u t h x e g s k n g i} {f j w g c s r}
14 {b f i c s u z t k} {c k q s j u i z o}
15 {n a f n u s w h y n s i q e w} {x g e g a s s h n}
16 {k s q e j n p} {t r j f t o e k k l m i}
17 {g d t u w r o p m n m n p h b o u} {h s w o s l j e}
18 {f l q y q q g e e x j r} {n b r r g e i r t x q k}
19 {f i r g o a w e p i l o a w} {e k r z t d g h g i b d i e m}
20 {l d u u f p y} {g o m m u x m g l j t t x x u}
21 {m c d k x i c z l} {m i a i e u h}
22 {w b f o c g x y j} {z d w x d f h i p}
23 {w u i u x t c h k i b} {b y k h b v r t g j}
24 {h f d j s w s b a p k} {a q y u z e y m m j q r}
25 {d i x y x x k i y f s d j h z p n} {l l q m e t c w g y h t s v g}
26 {g s q w t d k x g f m j p k y} {r m b x e l t d}
27 {j l s q u g y v e c l o} {m f l m m m h g x x l n c}
28 {c t j g v r s b z j} {l c f y d t q n}
29 {e x z y w i h l} {b n b x e y q e n u m}
30 {g y y h j b w r} {q b q f u s k c k g r}
31 {g u l x l b r c m z b u c} {k g t b x k x n t e z d h o}
32 {w g v l z f b z h p s c v h} {g e w v m h k r g w a r f q}
33 {c g n f u d o y o b} {e y o h x x y y i z s b h a j}
34 {v y h c q u u s q y x x k s q} {d n r m y k n t i r n w e}
35 {o u c x l e b t a} {y b a x y f z x r}
36 {x p h l j a a u u j h} {x o f s z m b c q p}
37 {k q t i c a q n m v v} {v r z e f m y o}
38 {r w t t t t r v v o e p g h} {l w x a g a u h y}
39 {o p v g v b a g o} {j t q c r b b g y z}
40 {f s o r o d t h q f x l} {r d b m k i f s t d l m y x j w}
41 {t m o t m f m f} {i p i q j v n v m b q}
42 {t x w a r l w d t b c o d o} {a h f h w z d n s}
43 {t u q c d g p q x j o l c x c} {m n t o z z j a y}
44 {v d i i k b f s z r v r z y} {g n q y s x x m b x c l w}
45 {p v v a c s z y e o l} {m v t u d k m k q b d c v z r}
46 {f y k l d r q w r s t r e} {h m v r r l r r t f q e x y}
47 {w l n l t y x} {n h s l a f c h u f l x x m v n o}
48 {t n v i k e b p z p d j j l i o} {i v z p g u e j s i k n h w d c}
49 {z v x p n l t a j c} {e j l e n c e t a d}
50 {w u b x u i v h a i y m m r p m s} {s r h d o g z y f f x e}
51 {d c c x b c a x g} {p r a j v u y}
52 {f w g r c o d l t u e z h i} {j l l s s b j m}
53 {p m t f k i x} {u v y a z g w v v m x h i}
54 {l c z g l o j i c d e b} {b f v y w u i b e i y}
55 {r h c x f x a d s} {z x y k f l r b q c v}
56 {v x x c y h z x b g m o q n c} {h n b i t g h a q b c o r u}
57 {d g l o h t b s b r} {n u e p t i m u}
58 {t d y e t d c w u o s w x f c h} {i o s v y b r d r}
59 {l b a p q n d r} {k d c c d n y q h g a o p e x}
60 {f r z v m p k r} {x x r i s b a g f c}
61 {s a z i e r f i w c n y v z t k s} {y y i r y n l s b w i e k n}
62 {n x p r e x q r m v i b y} {f o o z n b s r q j}
63 {y j s u j x o n r q t f} {f v k n v x u s o a d e f e}
64 {u s i l y c x q} {r k c h p c h b o s s u s p b}
65 {m p i o s h o} {s w h u n d m n q t y k b w c}
66 {l d f g m x x x o} {s w d d f b y j j h h t i y p j o}
67 {c b m h f n v w n h} {i r w i e x r w l z p x u g u l s}
68 {y a h u h i m a y q} {d d r x h e v q n z y c j}
69 {c x f d x o n p o b r t b l p l} {m i t k b x v f p t m l l y r o}
70 {u t l w w m s} {m f m o l t k o p e}
71 {f g q e l n d m z x q} {z s i i i m f w w f n g p e q}
72 {n l h a v u o d f j d e x} {v v s l f g d g r a j x i f z x}
73 {x v m v f i g q e w} {r y s j i k m j j e d g r n o i f}
74 {g d y n o h p s y q z j d w n h w} {x o d l t j i b r d o r y}
75 {p g b i u r b e q d v o a g w m k} {q y z s f q o h}
76 {u z a q u f i f f b} {b s p b a a d x r r i q f}
77 {w h h z t h p o a h h e e} {h w r p h k z v y f r x}
78 {c a r k i a p u x} {f w l p t e m l}
79 {q q u k o t r k z} {f b m c w p s s o z}
80 {t i g v y q s r x m r x z e f} {x o j w a u e y s j c b u p p r o}
81 {n j n h r l a r e o z w e} {v o r r j a v b}
82 {i f i d k w d n h} {o i d z i z l m w s b q v u}
83 {m d g q q b k b w f q q p p} {j m q f b y c i z k y q p l e a}
84 {m x o n y f g} {y c n x n q j i y c l h b r q z}
85 {v o z l n p c} {g n j n t b b x n c l d a g j v}
86 {z n a y f b t k k t d b z a v} {r p c n r u k u}
87 {b q t x z e c w} {q a o a l o a h i m j r}
88 {j f h o x x a z g b a f a m i b} {j z c z y x e x w t}
89 {t c t p r s u c q n} {z x l i k n f q l n t}
90 {w t d q j g m r f k n} {l e w f w w a l y q k i q t p c t}
91 {c b o k l i c b s j n m b l} {y f p q o w g}
92 {f y d j o q t c c q m f j s t} {f h e d y m o k}
93 {k x j r m a d o i z j} {r t t t f e b r x i v j v g o}
94 {s f e a e t i h h d q p z t q} {b k m k w h c}
95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p}
96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o}
97 {u h h k m n k} {u b v n u a o c}
98 {s p e t c z d f n w f} {l s f j b l c e s h}
99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o}
}
foreach {tn2 sql} {
1 {}
2 {BEGIN}
} {
reset_db
fts5_aux_test_functions db
do_execsql_test 1.$tn2.0 {
CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
}
execsql $sql
do_test 1.$tn2.1.1 {
foreach {id x y} $data {
execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
}
execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
} {}
#-------------------------------------------------------------------------
#
do_execsql_test 1.$tn2.integrity {
INSERT INTO xx(xx) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
foreach {tn expr} {
1.1 "a AND b"
1.2 "a OR b"
1.3 "o"
1.4 "b q"
1.5 "e a e"
1.6 "m d g q q b k b w f q q p p"
1.7 "l o o l v v k"
1.8 "a"
1.9 "b"
1.10 "c"
1.11 "no"
1.12 "L O O L V V K"
1.13 "a AND b AND c"
1.14 "x:a"
2.1 "x:a"
2.2 "y:a"
2.3 "x:b"
2.4 "y:b"
3.1 "{x}:a"
3.2 "{y}:a"
3.3 "{x}:b"
3.4 "{y}:b"
4.1 "{x y}:a"
4.2 "{y x}:a"
4.3 "{x x}:b"
4.4 "{y y}:b"
5.1 {{"x" "y"}:a}
5.2 {{"y" x}:a}
5.3 {{x "x"}:b}
5.4 {{"y" y}:b}
6.1 "b + q"
6.2 "e + a + e"
6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
6.4 "l + o + o + l + v + v + k"
6.5 "L + O + O + L + V + V + K"
7.1 "a+b AND c"
7.2 "d+c AND u"
7.3 "d+c AND u+d"
7.4 "a+b OR c"
7.5 "d+c OR u"
7.6 "d+c OR u+d"
8.1 "NEAR(a b)"
8.2 "NEAR(r c)"
8.2 { NEAR(r c, 5) }
8.3 { NEAR(r c, 3) }
8.4 { NEAR(r c, 2) }
8.5 { NEAR(r c, 0) }
8.6 { NEAR(a b c) }
8.7 { NEAR(a b c, 8) }
8.8 { x : NEAR(r c) }
8.9 { y : NEAR(r c) }
9.1 { NEAR(r c) }
9.2 { NEAR(r c, 5) }
9.3 { NEAR(r c, 3) }
9.4 { NEAR(r c, 2) }
9.5 { NEAR(r c, 0) }
9.6 { NEAR(a b c) }
9.7 { NEAR(a b c, 8) }
9.8 { x : NEAR(r c) }
9.9 { y : NEAR(r c) }
9.10 { x : "r c" }
9.11 { y : "r c" }
9.12 { a AND b }
9.13 { a AND b AND c }
9.14a { a }
9.14b { a OR b }
9.15 { a OR b AND c }
9.16 { c AND b OR a }
9.17 { c AND (b OR a) }
9.18 { c NOT (b OR a) }
9.19 { (c NOT b) OR (a AND d) }
} {
if {[fts5_expr_ok $expr xx]==0} {
do_test 1.$tn2.$tn.OMITTED { list } [list]
continue
}
set res [fts5_query_data $expr xx]
do_execsql_test 1.$tn2.$tn.[llength $res].asc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr
} $res
set res [fts5_query_data $expr xx DESC]
do_execsql_test 1.$tn2.$tn.[llength $res].desc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr ORDER BY 1 DESC
} $res
}
}
}
do_execsql_test 2.1 {
SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}
do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0}
do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2}
foreach {tn expr tclexpr} {
1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
} {
do_execsql_test 2.3.$tn {
SELECT fts5_expr_tcl($expr, 'N $x')
} [list $tclexpr]
}
finish_test
-243
View File
@@ -1,243 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the focus is on testing prefix queries, both with and
# without prefix indexes.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ad
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%);
INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is');
INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the');
} {}
foreach {tn match res} {
1 {c*} {1}
2 {i*} {3 2}
3 {t*} {3 1}
4 {r*} {3 1}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid DESC
} $res
}
foreach {tn match res} {
5 {c*} {1}
6 {i*} {2 3}
7 {t*} {1 3}
8 {r*} {1 3}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM yy WHERE yy MATCH $match
} $res
}
foreach {T create} {
2 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
3 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
4 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
5 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
} {
do_test $T.1 {
execsql { DROP TABLE IF EXISTS t1 }
execsql $create
} {}
do_test $T.1 {
foreach {rowid a b} {
0 {fghij uvwxyz klmn pq uvwx} {klmn f fgh uv fghij klmno}
1 {uv f abcd abcd fghi} {pq klm uv uv fgh uv a}
2 {klmn klm pqrs fghij uv} {f k uvw ab abcd pqr uv}
3 {ab pqrst a fghi ab pqr fg} {k klmno a fg abcd}
4 {abcd pqrst uvwx a fgh} {f klmno fghij kl pqrst}
5 {uvwxyz k abcde u a} {uv k k kl klmn}
6 {uvwxyz k klmn pqrst uv} {fghi pqrs abcde u k}
7 {uvwxy klmn u p pqrst fgh} {p f fghi abcd uvw kl uv}
8 {f klmno pqrst uvwxy pqrst} {uv abcde klm pq pqr}
9 {f abcde a uvwxyz pqrst} {fghij abc k uvwx pqr fghij uvwxy}
10 {ab uv f fg pqrst uvwxy} {fgh p uv k abc klm uvw}
11 {pq klmno a uvw abcde uvwxyz} {fghij pq uvwxyz pqr fghi}
12 {fgh u pq fgh uvw} {uvw pqr f uvwxy uvwx}
13 {uvwx klmn f fgh abcd pqr} {uvw k fg uv klm abcd}
14 {ab uvwx pqrst pqr uvwxyz pqrs} {uvwxyz abcde ab ab uvw abcde}
15 {abc abcde uvwxyz abc kl k pqr} {klm k k klmno u fgh}
16 {fghi abcd fghij uv uvwxyz ab uv} {klmn pqr a uvw fghi}
17 {abc pqrst fghi uvwx uvw klmn fghi} {ab fg pqr pqrs p}
18 {pqr kl a fghij fgh fg kl} {pqr uvwxyz uvw abcd uvwxyz}
19 {fghi fghi pqr kl fghi f} {klmn u u klmno klmno}
20 {abc pqrst klmno kl pq uvwxy} {abc k fghi pqrs klm}
21 {a pqr uvwxyz uv fghi a fgh} {abc pqrs pqrst pq klm}
22 {klm abc uvwxyz klm pqrst} {fghij k pq pqr u klm fghij}
23 {p klm uv p a a} {uvwxy klmn uvw abcde pq}
24 {uv fgh fg pq uvwxy u uvwxy} {pqrs a uvw p uvwx uvwxyz fg}
25 {fghij fghi klmn abcd pq kl} {fghi abcde pqrs abcd fgh uvwxy}
26 {pq fgh a abc klmno klmn} {fgh p k p fg fghij}
27 {fg pq kl uvwx fghij pqrst klmn} {abcd uvw abcd fghij f fghij}
28 {uvw fghi p fghij pq fgh uvwx} {k fghij abcd uvwx pqr fghi}
29 {klm pq abcd pq f uvwxy} {pqrst p fghij pqr p}
30 {ab uvwx fg uvwx klmn klm} {klmn klmno fghij klmn klm}
31 {pq k pqr abcd a pqrs} {abcd abcd uvw a abcd klmno ab}
32 {pqrst u abc pq klm} {abc kl uvwxyz fghij u fghi p}
33 {f uvwxy u k f uvw uvwx} {pqrs uvw fghi fg pqrst klm}
34 {pqrs pq fghij uvwxyz pqr} {ab abc abc uvw f pq f}
35 {uvwxy ab uvwxy klmno kl pqrs} {abcde uvw pqrs uvwx k k}
36 {uvwxyz k ab abcde abc uvw} {uvw abcde uvw klmn uv klmn}
37 {k kl uv abcde uvwx fg u} {u abc uvwxy k fg abcd}
38 {fghi pqrst fghi pqr pqrst uvwx} {u uv uvwx fghi abcde}
39 {k pqrst k uvw fg pqrst fghij} {uvwxy ab kl klmn uvwxyz abcde}
40 {fg uvwxy pqrs klmn uvwxyz klm p} {k uv ab fghij fgh k pqrs}
41 {uvwx abc f pq uvwxy k} {ab uvwxyz abc f fghij}
42 {uvwxy klmno uvwxyz uvwxyz pqrst} {uv kl kl klmno k f abcde}
43 {abcde ab pqrs fg f fgh} {abc fghij fghi k k}
44 {uvw abcd a ab pqrst klmn fg} {pqrst u uvwx pqrst fghij f pqrst}
45 {uvwxy p kl uvwxyz ab pqrst fghi} {abc f pqr fg a k}
46 {u p f a fgh} {a kl pq uv f}
47 {pqrs abc fghij fg abcde ab a} {p ab uv pqrs kl fghi abcd}
48 {abcde uvwxy pqrst uv abc pqr uvwx} {uvwxy klm uvwxy uvwx k}
49 {fgh klm abcde klmno u} {a f fghij f uvwxyz abc u}
50 {uv uvw uvwxyz uvwxyz uv ab} {uvwx pq fg u k uvwxy}
51 {uvwxy pq p kl fghi} {pqrs fghi pqrs abcde uvwxyz ab}
52 {pqr p uvwxy kl pqrs klmno fghij} {ab abcde abc pqrst pqrs uv}
53 {fgh pqrst p a klmno} {ab ab pqrst pqr kl pqrst}
54 {abcd klm ab uvw a fg u} {f pqr f abcd uv}
55 {u fg uvwxyz k uvw} {abc pqrs f fghij fg pqrs uvwxy}
56 {klm fg p fghi fg a} {uv a fghi uvwxyz a fghi}
57 {uvwxy k abcde fgh f fghi} {f kl klmn f fghi klm}
58 {klm k fgh uvw fgh fghi} {klmno uvwx u pqrst u}
59 {fghi pqr pqrst p uvw fghij} {uv pqrst pqrs pq fghij klm}
60 {uvwx klm uvwxy uv klmn} {p a a abc klmn ab k}
61 {uvwxy uvwx klm uvwx klm} {pqrs ab ab uvwxyz fg}
62 {kl uv uv uvw fg kl k} {abcde uvw fgh uvwxy klm}
63 {a abc fgh u klm abcd} {fgh pqr uv klmn fghij}
64 {klmn k klmn klmno pqrs pqr} {fg kl abcde klmno uvwxy kl pq}
65 {uvwxyz klm fghi abc abcde kl} {uvwxy uvw uvwxyz uvwxyz pq pqrst}
66 {pq klm abc pqrst fgh f} {u abcde pqrst abcde fg}
67 {u pqrst kl u uvw klmno} {u pqr pqrs fgh u p}
68 {abc fghi uvwxy fgh k pq} {uv p uvwx uvwxyz ab}
69 {klmno f uvwxyz uvwxy klmn fg ab} {fgh kl a pqr abcd pqr}
70 {fghi pqrst pqrst uv a} {uvwxy k p uvw uvwx a}
71 {a fghij f p uvw} {klm fg abcd abcde klmno pqrs}
72 {uv uvwx uvwx uvw klm} {uv fghi klmno uvwxy uvw}
73 {kl uvwxy ab f pq klm u} {uvwxy klmn klm abcd pq fg k}
74 {uvw pqrst abcd uvwxyz ab} {fgh fgh klmn abc pq}
75 {uvwxyz klm pq abcd klmno pqr uvwxyz} {kl f a fg pqr klmn}
76 {uvw uvwxy pqr k pqrst kl} {uvwxy abc uvw uvw u}
77 {fgh klm u uvwxyz f uvwxy abcde} {uv abcde klmno u u ab}
78 {klmno abc pq pqr fgh} {p uv abcd fgh abc u k}
79 {fg pqr uvw pq uvwx} {uv uvw fghij pqrs fg p}
80 {abcd pqrs uvwx uvwxy uvwx} {u uvw pqrst pqr abcde pqrs kl}
81 {uvwxyz klm pq uvwxy fghij} {p pq klm fghij u a a}
82 {uvwx k uvwxyz klmno pqrst kl} {abcde p f pqrst abcd uvwxyz p}
83 {abcd abcde klm pqrst uvwxyz} {uvw pqrst u p uvwxyz a pqrs}
84 {k klm abc uv uvwxy klm klmn} {k abc pqr a abc p kl}
85 {klmn abcd pqrs p pq klm a} {klmn kl ab uvw pq}
86 {klmn a pqrs abc uvw pqrst} {a pqr kl klm a k f}
87 {pqrs ab uvwx uvwxy a pqr f} {fg klm uvwx pqr pqr}
88 {klmno ab k kl u uvwxyz} {uv kl uvw fghi uv uvw}
89 {pq fghi pqrst klmn uvwxy abc pqrs} {fg f f fg abc abcde klm}
90 {kl a k fghi uvwx fghi u} {ab uvw pqr fg a p abc}
91 {uvwx pqrs klmno ab fgh uvwx} {pqr uvwx abc kl f klmno kl}
92 {fghij pq pqrs fghij f pqrst} {u abcde fg pq pqr fgh k}
93 {fgh u pqrs abcde klmno abc} {abc fg pqrst pqr abcde}
94 {uvwx p abc f pqr p} {k pqrs kl klm abc fghi klm}
95 {kl p klmno uvwxyz klmn} {fghi ab a fghi pqrs kl}
96 {pqr fgh pq uvwx a} {uvw klm klmno fg uvwxy uvwx}
97 {fg abc uvwxyz fghi pqrst pq} {abc k a ab abcde f}
98 {uvwxy fghi uvwxy u abcde abcde uvw} {klmn uvwx pqrs uvw uvwxy abcde}
99 {pq fg fghi uvwx uvwx fghij uvwxy} {klmn klmn f abc fg a}
} {
execsql {
INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b);
}
}
} {}
proc prefix_query {prefixlist} {
set ret [list]
db eval {SELECT rowid, a, b FROM t1 ORDER BY rowid DESC} {
set bMatch 1
foreach pref $prefixlist {
if { [lsearch -glob $a $pref]<0 && [lsearch -glob $b $pref]<0 } {
set bMatch 0
break
}
}
if {$bMatch} { lappend ret $rowid }
}
return $ret
}
do_execsql_test $T.integrity {
INSERT INTO t1(t1) VALUES('integrity-check');
}
foreach {bAsc sql} {
1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC}
} {
foreach {tn prefix} {
1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*}
6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*}
11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*}
16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*}
21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
27 {x*}
28 {a f*} 29 {a* f*} 30 {a* fghij*}
} {
set res [prefix_query $prefix]
if {$bAsc} {
set res [lsort -integer -increasing $res]
}
set n [llength $res]
do_execsql_test $T.$bAsc.$tn.$n $sql $res
}
}
catchsql COMMIT
}
}
finish_test
-311
View File
@@ -1,311 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ae
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 1.1 {
INSERT INTO t1 VALUES('hello', 'world');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1}
do_execsql_test 1.2 {
INSERT INTO t1 VALUES('world', 'hello');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2}
do_execsql_test 1.3 {
INSERT INTO t1 VALUES('world', 'world');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2}
do_execsql_test 1.4.1 {
INSERT INTO t1 VALUES('hello', 'hello');
}
do_execsql_test 1.4.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2 4}
fts5_aux_test_functions db
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}
do_execsql_test 2.1 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'm' ORDER BY rowid;
} {
1 {0.0.5 0.1.0 0.1.2}
2 {0.0.7 0.1.5}
}
do_execsql_test 2.2 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'u OR q' ORDER BY rowid;
} {
1 {0.0.0}
2 {1.0.2 1.0.10}
}
if {[detail_is_full]} {
do_execsql_test 2.3 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
1 {0.1.3 0.1.7}
}
}
#-------------------------------------------------------------------------
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
INSERT INTO t3 VALUES( 'r c', '');
}
if {[detail_is_full]} {
do_execsql_test 3.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
}
do_execsql_test 3.2 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
2 {0.0.0 1.0.1}
}
}
do_execsql_test 3.3 {
INSERT INTO t3
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
SELECT rowid, fts5_test_poslist(t3)
FROM t3 WHERE t3 MATCH 'a OR b AND c';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2}
3 0.0.5
}
#-------------------------------------------------------------------------
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t4
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}
do_execsql_test 4.1 {
SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
1 0.0.5
}
#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#
reset_db
fts5_aux_test_functions db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
INSERT INTO t5 VALUES('', 'a');
INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {1 0}
2 {0 1}
1 {4 6}
}
do_execsql_test 5.3 {
SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {a {}}
2 {{} a}
1 {{a b c d} {e f g h i j}}
}
do_execsql_test 5.4 {
SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {5 7}
2 {5 7}
1 {5 7}
}
do_execsql_test 5.5 {
INSERT INTO t5 VALUES('x y z', 'v w x y z');
SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {8 12}
2 {8 12}
1 {8 12}
}
#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}
do_execsql_test 6.2 {
SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*'
} {
1 {{there are more} {things in heaven and earth}}
2 {{horatio than are} {dreamt of in your philosophy}}
}
#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%);
}
do_test 7.2 {
foreach {x y} {
{q i b w s a a e l o} {i b z a l f p t e u}
{b a z t a l o x d i} {b p a d b f h d w y}
{z m h n p p u i e g} {v h d v b x j j c z}
{a g i m v a u c b i} {p k s o t l r t b m}
{v v c j o d a s c p} {f f v o k p o f o g}
} {
execsql {INSERT INTO t7 VALUES($x, $y)}
}
execsql { SELECT count(*) FROM t7 }
} {5}
foreach {tn q res} {
1 a {{4 2}}
2 b {{3 4}}
3 c {{2 1}}
4 d {{2 2}}
5 {a AND b} {{4 2} {3 4}}
6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
do_execsql_test 7.3.$tn {
SELECT fts5_test_queryphrase(t7) FROM t7 WHERE t7 MATCH $q LIMIT 1
} [list $res]
}
do_execsql_test 7.4 {
SELECT fts5_test_rowcount(t7) FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}
#do_execsql_test 7.4 {
# SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
#} {5 5 5 5}
#
#-------------------------------------------------------------------------
#
do_test 8.1 {
execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) }
foreach {rowid x y} {
0 {A o} {o o o C o o o o o o o o}
1 {o o B} {o o o C C o o o o o o o}
2 {A o o} {o o o o D D o o o o o o}
3 {o B} {o o o o o D o o o o o o}
4 {E o G} {H o o o o o o o o o o o}
5 {F o G} {I o J o o o o o o o o o}
6 {E o o} {H o J o o o o o o o o o}
7 {o o o} {o o o o o o o o o o o o}
9 {o o o} {o o o o o o o o o o o o}
} {
execsql { INSERT INTO t8(rowid, x, y) VALUES($rowid, $x, $y) }
}
} {}
foreach {tn q res} {
1 {a} {0 2}
2 {b} {3 1}
3 {c} {1 0}
4 {d} {2 3}
5 {g AND (e OR f)} {5 4}
6 {j AND (h OR i)} {5 6}
} {
do_execsql_test 8.2.$tn.1 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8);
} $res
do_execsql_test 8.2.$tn.2 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank;
} $res
do_execsql_test 8.2.$tn.3 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank;
} $res
}
#-------------------------------------------------------------------------
# Test xPhraseCount() for some different queries.
#
do_test 9.1 {
execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) }
foreach x {
"a b c" "d e f"
} {
execsql { INSERT INTO t9 VALUES($x) }
}
} {}
foreach {tn q cnt} {
1 {a AND b} 2
2 {a OR b} 2
3 {a OR b OR c} 3
4 {NEAR(a b)} 2
} {
do_execsql_test 9.2.$tn {
SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
} $cnt
}
}
finish_test
-190
View File
@@ -1,190 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the tests in this file focus on the built-in
# snippet() function.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5af
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
proc do_snippet_test {tn doc match res} {
uplevel #0 [list set v1 $doc]
uplevel #0 [list set v2 $match]
do_execsql_test $tn.1 {
DELETE FROM t1;
INSERT INTO t1 VALUES($v1, NULL);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
do_execsql_test $tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $v1);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
do_execsql_test $tn.3 {
DELETE FROM t1;
INSERT INTO t1 VALUES($v1, NULL);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2
ORDER BY rank DESC;
} [list $res]
}
foreach {tn doc res} {
1.1 {X o o o o o o} {[X] o o o o o o}
1.2 {o X o o o o o} {o [X] o o o o o}
1.3 {o o X o o o o} {o o [X] o o o o}
1.4 {o o o X o o o} {o o o [X] o o o}
1.5 {o o o o X o o} {o o o o [X] o o}
1.6 {o o o o o X o} {o o o o o [X] o}
1.7 {o o o o o o X} {o o o o o o [X]}
2.1 {X o o o o o o o} {[X] o o o o o o...}
2.2 {o X o o o o o o} {o [X] o o o o o...}
2.3 {o o X o o o o o} {o o [X] o o o o...}
2.4 {o o o X o o o o} {o o o [X] o o o...}
2.5 {o o o o X o o o} {o o o o [X] o o...}
2.6 {o o o o o X o o} {o o o o o [X] o...}
2.7 {o o o o o o X o} {o o o o o o [X]...}
2.8 {o o o o o o o X} {...o o o o o o [X]}
2.9 {o o o o o o o X o} {...o o o o o [X] o}
2.10 {o o o o o o o X o o} {...o o o o [X] o o}
2.11 {o o o o o o o X o o o} {...o o o [X] o o o}
2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
3.1 {X o o o o o o o o} {[X] o o o o o o...}
3.2 {o X o o o o o o o} {o [X] o o o o o...}
3.3 {o o X o o o o o o} {o o [X] o o o o...}
3.4 {o o o X o o o o o} {o o o [X] o o o...}
3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
6.1 {X o o o X o o o} {[X] o o o [X] o o...}
6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
{...o o [X] [X] [X] o o...}
8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o}
{...o o [X] o o o o...}
8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
{o o o o [X] o o...}
} {
do_snippet_test 1.$tn $doc X $res
}
if {[detail_is_full]} {
foreach {tn doc res} {
1.1 {X Y o o o o o} {[X Y] o o o o o}
1.2 {o X Y o o o o} {o [X Y] o o o o}
1.3 {o o X Y o o o} {o o [X Y] o o o}
1.4 {o o o X Y o o} {o o o [X Y] o o}
1.5 {o o o o X Y o} {o o o o [X Y] o}
1.6 {o o o o o X Y} {o o o o o [X Y]}
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_snippet_test 2.$tn $doc "X + Y" $res
}
}
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
} {
{1 2 3 4 5...}
}
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE p1 USING fts5(a, b);
INSERT INTO p1 VALUES(
'x a a a a a a a a a a',
'a a a a a a a a a a a a a a a a a a a x'
);
}
do_execsql_test 5.1 {
SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
} {{[x] a a a a a...}}
do_execsql_test 5.2 {
SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
} {{[x] a a a a a}}
do_execsql_test 5.3 {
SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
} {{x] a a a a a...}}
do_execsql_test 5.4 {
SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
} {{[x a a a a a...}}
} ;# foreach_detail_mode
finish_test
-144
View File
@@ -1,144 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ag
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# This file attempts to verify that the extension APIs work with
# "ORDER BY rank" queries. This is done by comparing the results of
# the fts5_test() function when run with queries of the form:
#
# ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC]
#
# and
#
# ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
#
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%);
}
do_test 1.1 {
foreach {x y z} {
{j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w}
{m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v}
{m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n}
{j e m v k p e c j m} {m p v z d x l n i a} {v p u p m t p q i f}
{v r w l e e t d z p} {c s b w k m n k o u} {w g y f v w v w v p}
{k d g o u j p z n o} {t g e q l z i g b j} {f i q q j y h b g h}
{j s w x o t j b t m} {v a v v r t x c q a} {r t k x w u l h a g}
{j y b i u d e m d w} {y s o j h i n a u p} {n a g b u c w e b m}
{b c k s c w j p w b} {m o c o w o b d q q} {n t y o y z y r z e}
{p n q l e l h z q c} {n s e i h c v b b u} {m p d i t a o o f f}
{k c o n v e z l b m} {s m n i n s d e s u} {t a u e q d a o u c}
{h d t o i a g b b p} {k x c i g f g b b k} {x f i v n a n n j i}
{f z k r b u s k z e} {n z v z w l e r h t} {t i s v v a v p n s}
{k f e c t z r e f d} {f m g r c w q k b v} {v y s y f r b f e f}
{z r c t d q q h x b} {u c g z n z u v s s} {y t n f f x b f d x}
{u n p n u t i m e j} {p j j d m f k p m z} {d o l v c o e a h w}
{h o q w t f v i c y} {c q u n r z s l l q} {z x a q w s b w s y}
{y m s x k i m n x c} {b i a n v h z n k a} {w l q p b h h g d y}
{z v s j f p v l f w} {c s b i z e k i g c} {x b v d w j f e d z}
{r k k j e o m k g b} {h b d c h m y b t u} {u j s h k z c u d y}
{v h i v s y z i k l} {d t m w q w c a z p} {r s e s x v d w k b}
{u r e q j y h o o s} {x x z r x y t f j s} {k n h x i i u e c v}
{q l f d a p w l q o} {y z q w j o p b o v} {s u h z h f d f n l}
{q o e o x x l g q i} {j g m h q q w c d b} {o m d h w a g b f n}
{m x k t s s y l v a} {j x t c a u w b w g} {n f j b v x y p u t}
{u w k a q b u w k w} {a h j u o w f s k p} {j o f s h y t j h g}
{x v b l m t l m h l} {t p y i y i q b q a} {k o o z w a c h c f}
{j g c d k w b d t v} {a k v c m a v h v p} {i c a i j g h l j h}
{l m v l c z j b p b} {z p z f l n k i b a} {j v q k g i x g i b}
{m c i w u z m i s z} {i z r f n l q z k w} {x n b p b q r g i z}
{d g i o o x l f x d} {r t m f b n q y c b} {i u g k w x n m p o}
{t o s i q d z x d t} {v a k s q z j c o o} {z f n n r l y w v v}
{w k h d t l j g n n} {r z m v y b l n c u} {v b v s c l n k g v}
{m a g r a b u u n z} {u y l h v w v k b f} {x l p g i s j f x v}
{v s g x k z a k a r} {l t g v j q l k p l} {f h n a x t v s t y}
{z u v u x p s j y t} {g b q e e g l n w g} {e n p j i g j f u r}
{q z l t w o l m p e} {t s g h r p r o t z} {y b f a o n u m z g}
{d t w n y b o g f o} {d a j e r l g g s h} {d z e l w q l t h f}
{f l u w q v x j a h} {f n u l l d m h h w} {d x c c e r o d q j}
{b y f q s q f u l g} {u z w l f d b i a g} {m v q b g u o z e z}
{h z p t s e x i v m} {l h q m e o x x x j} {e e d n p r m g j f}
{k h s g o n s d a x} {u d t t s j o v h a} {z r b a e u v o e s}
{m b b g a f c p a t} {w c m j o d b l g e} {f p j p m o s y v j}
{c r n h d w c a b l} {s g e u s d n j b g} {b o n a x a b x y l}
{r h u x f c d z n o} {x y l g u m i i w d} {t f h b z v r s r g}
{t i o r b v g g p a} {d x l u q k m o s u} {j f h t u n z u k m}
{g j t y d c n j y g} {w e s k v c w i g t} {g a h r g v g h r o}
{e j l a q j g i n h} {d z k c u p n u p p} {t u e e v z v r r g}
{l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
{i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
} {
execsql { INSERT INTO t1 VALUES($x, $y, $z) }
}
set {} {}
} {}
fts5_aux_test_functions db
proc do_fts5ag_test {tn E} {
set q1 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
set q2 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}
set res [execsql $q1]
set expected [execsql $q2]
uplevel [list do_test $tn.1 [list set {} $res] $expected]
append q1 " DESC"
append q2 " DESC"
set res [execsql $q1]
set expected [execsql $q2]
uplevel [list do_test $tn.2 [list set {} $res] $expected]
}
foreach {tn expr} {
2.1 a
2.2 b
2.3 c
2.4 d
3.0 {a AND b}
3.1 {a OR b}
3.2 {b OR c AND d}
} {
do_fts5ag_test $tn $expr
}
if {[detail_is_full]} {
foreach {tn expr} {
4.1 {"m m"}
4.2 {e + s}
4.3 {NEAR(c d)}
} {
do_fts5ag_test $tn $expr
}
}
} ;# foreach_detail_mode
finish_test
-169
View File
@@ -1,169 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ah
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#
set Y [list]
set W [list]
do_test 1.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
set v {w w w w w w w w w w w w w w w w w w w w}
execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
for {set i 1} {$i <= 10000} {incr i} {
set v {x x x x x x x x x x x x x x x x x x x x}
if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
execsql { INSERT INTO t1 VALUES($v) }
}
set v {w w w w w w w w w w w w w w w w w w w w}
execsql { INSERT INTO t1 VALUES($v) }
} {}
do_execsql_test 1.1.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w'
} [lsort -integer -incr $W]
do_execsql_test 1.1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'x* AND w*'
} [lsort -integer -incr $W]
do_execsql_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -incr $Y]
do_execsql_test 1.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
proc reads {} {
db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'}
}
proc execsql_reads {sql} {
set nRead [reads]
execsql $sql
expr [reads] - $nRead
}
do_test 1.4 {
set nRead [reads]
execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
set nReadX [expr [reads] - $nRead]
#puts -nonewline "(nReadX=$nReadX)"
if {[detail_is_full]} { set expect 1000 }
if {[detail_is_col]} { set expect 250 }
if {[detail_is_none]} { set expect 80 }
expr $nReadX>$expect
} {1}
do_test 1.5 {
set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }]
set bwd [execsql_reads {
SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC
}]
expr {$bwd < $fwd + 12}
} {1}
foreach {tn q res} "
1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W]
2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W]
3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W]
4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y]
" {
if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue
if {[detail_is_full]} { set ratio 8 }
if {[detail_is_col]} { set ratio 4 }
if {[detail_is_none]} { set ratio 2 }
do_test 1.6.$tn.1 {
set n [execsql_reads $q]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / $ratio)}
} {1}
do_test 1.6.$tn.2 {
set n [execsql_reads "$q ORDER BY rowid DESC"]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / $ratio)}
} {1}
do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res]
}
#-------------------------------------------------------------------------
# Now test that adding range constraints on the rowid field reduces the
# number of pages loaded from disk.
#
foreach {tn fraction tail cnt} {
1 0.6 {rowid > 5000} 5000
2 0.2 {rowid > 9000} 1000
3 0.2 {rowid < 1000} 999
4 0.2 {rowid BETWEEN 4000 AND 5000} 1001
5 0.6 {rowid >= 5000} 5001
6 0.2 {rowid >= 9000} 1001
7 0.2 {rowid <= 1000} 1000
8 0.6 {rowid > '5000'} 5000
9 0.2 {rowid > '9000'} 1000
10 0.1 {rowid = 444} 1
} {
set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail"
set n [execsql_reads $q]
set ret [llength [execsql $q]]
# Because the position lists for 'x' are quite long in this db, the
# advantage is a bit smaller in detail=none mode. Update $fraction to
# reflect this.
if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] }
do_test "1.7.$tn.asc.(n=$n ret=$ret)" {
expr {$n < ($fraction*$nReadX) && $ret==$cnt}
} {1}
set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC"
set n [execsql_reads $q]
set ret [llength [execsql $q]]
do_test "1.7.$tn.desc.(n=$n ret=$ret)" {
expr {$n < 2*$fraction*$nReadX && $ret==$cnt}
} {1}
}
do_execsql_test 1.8.1 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text';
} {10000}
do_execsql_test 1.8.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
} {10000}
} ;# foreach_detail_mode
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
finish_test
-57
View File
@@ -1,57 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, it tests transactions and savepoints
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ai
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%);
} {}
do_execsql_test 1.1 {
BEGIN;
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('d e f');
SAVEPOINT one;
INSERT INTO t1 VALUES('g h i');
SAVEPOINT two;
INSERT INTO t1 VALUES('j k l');
ROLLBACK TO one;
INSERT INTO t1 VALUES('m n o');
SAVEPOINT two;
INSERT INTO t1 VALUES('p q r');
RELEASE one;
SAVEPOINT one;
INSERT INTO t1 VALUES('s t u');
ROLLBACK TO one;
COMMIT;
}
do_execsql_test 1.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
finish_test
-68
View File
@@ -1,68 +0,0 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, this tests that, provided the amount of data remains
# constant, the FTS index does not grow indefinitely as rows are inserted
# and deleted,
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aj
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc doc {} {
set dict [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
set res [list]
for {set i 0} {$i < 20} {incr i} {
lappend res [lindex $dict [expr int(rand() * 26)]]
}
set res
}
proc structure {} {
set val [db one {SELECT fts5_decode(rowid,block) FROM t1_data WHERE rowid=10}]
foreach lvl [lrange $val 1 end] {
lappend res [expr [llength $lvl]-2]
}
set res
}
expr srand(0)
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
}
for {set iTest 0} {$iTest < 50000} {incr iTest} {
if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } }
set new [doc]
execsql { INSERT INTO t1 VALUES($new) }
if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] }
if {0==($iTest % 1000)} {
set sz [db one {SELECT count(*) FROM t1_data}]
set s [structure]
do_execsql_test 1.$iTest.$sz.{$s} {
INSERT INTO t1(t1) VALUES('integrity-check')
}
}
}
do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
finish_test

Some files were not shown because too many files have changed in this diff Show More