Compare commits

..

3 Commits

Author SHA1 Message Date
drh 739aab893a Version 3.2.8 (CVS 2835)
FossilOrigin-Name: e61382aed45a83ad688a0461948f9cd052a527fa
2005-12-19 17:26:59 +00:00
drh 6a42c8f368 Bump the version number on the 3.2 branch to 3.2.8. (CVS 2830)
FossilOrigin-Name: d8e74af5b081a074173c470e9f2115be9b095777
2005-12-19 16:34:31 +00:00
drh 523de6a830 Backport the fix for ticket #1567 into the 3.2 branch. (CVS 2829)
FossilOrigin-Name: 33397742282425351157a92785f83f10ad564af7
2005-12-19 16:29:07 +00:00
990 changed files with 78868 additions and 420705 deletions
-138
View File
@@ -1,138 +0,0 @@
#!/usr/make
#
# Makefile for SQLITE
#
# This is a template makefile for SQLite. Most people prefer to
# use the autoconf generated "configure" script to generate the
# makefile automatically. But that does not work for everybody
# and in every situation. If you are having problems with the
# "configure" script, you might want to try this makefile as an
# alternative. Create a copy of this file, edit the parameters
# below and type "make".
#
#### The directory where to find the mingw32ce tools
MINGW32CE = /opt/mingw32ce/bin
#### The target prefix of the mingw32ce tools
TARGET = arm-wince-mingw32ce
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = ../sqlite
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DTHREADSAFE=1
#THREADSAFE = -DTHREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
#TLIBS = -lrt # fdatasync on Solaris 8
TLIBS =
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_WIN=1 -D_WIN32_WCE=1
#OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .exe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(MINGW32CE)/$(TARGET)-gcc -O2
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#### Tools used to build a static library.
#
#AR = ar cr
#AR = /opt/mingw/bin/i386-mingw32-ar cr
AR = $(MINGW32CE)/$(TARGET)-ar cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
RANLIB = $(MINGW32CE)/$(TARGET)-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(MINGW32CE)/$(TARGET)-gcc -shared
SO = dll
SHPREFIX =
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
# You should not have to change anything below this line
###############################################################################
include $(TOP)/main.mk
+351 -637
View File
File diff suppressed because it is too large Load Diff
+7 -23
View File
@@ -41,11 +41,6 @@ THREADSAFE = -DTHREADSAFE=0
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
#TLIBS = -lrt # fdatasync on Solaris 8
TLIBS =
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
@@ -58,7 +53,6 @@ TLIBS =
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1
OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
@@ -83,31 +77,21 @@ AR = ar cr
RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
MKSHLIB = gcc -shared
SO = so
SHPREFIX = lib
# SO = dll
# SHPREFIX =
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.4linux
#TCL_FLAGS = -I/home/drh/tcltk/8.4win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
LIBTCL = /home/drh/tcltk/8.4linux/libtcl8.4g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.4win/libtcl84s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
#TCLOBJ =
TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
@@ -118,10 +102,10 @@ READLINE_FLAGS =
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#### Should the database engine assume text is coded as UTF-8 or iso8859?
#
# NAWK = nawk
NAWK = awk
# ENCODING = UTF8
ENCODING = ISO8859
# You should not have to change anything below this line
###############################################################################
-663
View File
@@ -1,663 +0,0 @@
#!/usr/make
#
# Makefile for SQLITE on VxWorks
ifeq ($(FORCPU),)
FORCPU = SH32gnule
endif
TOOL_FAMILY = gnu
include $(WIND_USR)/tool/gnu/make.$(FORCPU)
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = .
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DSQLITE_THREADSAFE=1
#THREADSAFE = -DSQLITE_THREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
ifeq ($(CPU),SH32)
# for SH4 shared library
TLIBS_SHARED += -L$(WIND_USR)/lib/sh/SH32/commonle/PIC
else
# for all other CPUs shared library
TLIBS_SHARED += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
endif
# for static library
TLIBS += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_UNIX=1 $(THREADSAFE)
OPTS += -DSQLITE_OMIT_LOAD_EXTENSION=1
OPTS += -DSQLITE_ENABLE_LOCKING_STYLE=1
OPTS += -DSQLITE_THREAD_OVERRIDE_LOCK=0
OPTS += -DSQLITE_ENABLE_COLUMN_METADATA=1
OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .vxe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(CC) $(DEFINE_CC) -O2 -g -mrtp $(CC_ARCH_SPEC) -D_REENTRANT=1 -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
TCC += -I$(WIND_USR)/h -I$(WIND_USR)/h/wrn/coreip
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#TCC_SHARED = $(TCC) -fPIC
TCC_SHARED = $(TCC)
#### Tools used to build a static library.
#
#ARX = ar cr
#ARX = /opt/mingw/bin/i386-mingw32-ar cr
AR += cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(CC) $(DEFINE_CC) -mrtp -shared $(CC_ARCH_SPEC) -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
SO = so
SHPREFIX = lib
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
#### Pasted and adapted main.mk file
###############################################################################
# The following macros should be defined before this script is
# invoked:
#
# TOP The toplevel directory of the source tree. This is the
# directory that contains this "Makefile.in" and the
# "configure.in" script.
#
# BCC C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
# THREADLIB Specify any extra linker options needed to make the library
# thread safe
#
# OPTS Extra compiler command-line options.
#
# EXE The suffix to add to executable files. ".exe" for windows
# and "" for Unix.
#
# TCC C Compiler and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
# AR Tools used to build a static library.
# RANLIB
#
# TCL_FLAGS Extra compiler options needed for programs that use the
# TCL library.
#
# LIBTCL Linker options needed to link against the TCL library.
#
# READLINE_FLAGS Compiler options needed for programs that use the
# readline() library.
#
# LIBREADLINE Linker options needed by programs using readline() must
# link against.
#
# NAWK Nawk compatible awk program. Older (obsolete?) solaris
# systems need this to avoid using the original AT&T AWK.
#
# Once the macros above are defined, the rest of this make script will
# build the SQLite library and testing tools.
################################################################################
# This is how we compile
#
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX_SHARED = $(TCC_SHARED) $(OPTS) -I. -I$(TOP)/src -I$(TOP) \
-I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 \
-I$(TOP)/ext/async
# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o \
backup.o bitvec.o btmutex.o btree.o build.o \
callback.o complete.o date.o delete.o expr.o fault.o \
fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
fts3_tokenizer.o fts3_tokenizer1.o \
func.o global.o hash.o \
icu.o insert.o journal.o legacy.o loadext.o \
main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
memjournal.o \
mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \
notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \
pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
update.o util.o vacuum.o \
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \
walker.o where.o utf.o vtab.o
# All of the source code files.
#
SRC = \
$(TOP)/src/alter.c \
$(TOP)/src/analyze.c \
$(TOP)/src/attach.c \
$(TOP)/src/auth.c \
$(TOP)/src/backup.c \
$(TOP)/src/bitvec.c \
$(TOP)/src/btmutex.c \
$(TOP)/src/btree.c \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/build.c \
$(TOP)/src/callback.c \
$(TOP)/src/complete.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/delete.c \
$(TOP)/src/expr.c \
$(TOP)/src/fault.c \
$(TOP)/src/func.c \
$(TOP)/src/global.c \
$(TOP)/src/hash.c \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/journal.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
$(TOP)/src/malloc.c \
$(TOP)/src/mem0.c \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
$(TOP)/src/mutex_os2.c \
$(TOP)/src/mutex_unix.c \
$(TOP)/src/mutex_w32.c \
$(TOP)/src/notify.c \
$(TOP)/src/os.c \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_os2.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
$(TOP)/src/pager.h \
$(TOP)/src/parse.y \
$(TOP)/src/pcache.c \
$(TOP)/src/pcache.h \
$(TOP)/src/pcache1.c \
$(TOP)/src/pragma.c \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
$(TOP)/src/resolve.c \
$(TOP)/src/rowset.c \
$(TOP)/src/select.c \
$(TOP)/src/status.c \
$(TOP)/src/shell.c \
$(TOP)/src/sqlite.h.in \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/trigger.c \
$(TOP)/src/utf.c \
$(TOP)/src/update.c \
$(TOP)/src/util.c \
$(TOP)/src/vacuum.c \
$(TOP)/src/vdbe.c \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeapi.c \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
$(TOP)/src/walker.c \
$(TOP)/src/where.c
# Source code for extensions
#
SRC += \
$(TOP)/ext/fts1/fts1.c \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.c \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_porter.c \
$(TOP)/ext/fts1/fts1_tokenizer.h \
$(TOP)/ext/fts1/fts1_tokenizer1.c
SRC += \
$(TOP)/ext/fts2/fts2.c \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.c \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_icu.c \
$(TOP)/ext/fts2/fts2_porter.c \
$(TOP)/ext/fts2/fts2_tokenizer.h \
$(TOP)/ext/fts2/fts2_tokenizer.c \
$(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.c \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_icu.c \
$(TOP)/ext/fts3/fts3_porter.c \
$(TOP)/ext/fts3/fts3_tokenizer.h \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_tokenizer1.c
SRC += \
$(TOP)/ext/icu/sqliteicu.h \
$(TOP)/ext/icu/icu.c
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
# Generated source code files
#
SRC += \
keywordhash.h \
opcodes.c \
opcodes.h \
parse.c \
parse.h \
sqlite3.h
# Source code to the test files.
#
TESTSRC = \
$(TOP)/src/test1.c \
$(TOP)/src/test2.c \
$(TOP)/src/test3.c \
$(TOP)/src/test4.c \
$(TOP)/src/test5.c \
$(TOP)/src/test6.c \
$(TOP)/src/test7.c \
$(TOP)/src/test8.c \
$(TOP)/src/test9.c \
$(TOP)/src/test_autoext.c \
$(TOP)/src/test_async.c \
$(TOP)/src/test_backup.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_func.c \
$(TOP)/src/test_hexio.c \
$(TOP)/src/test_journal.c \
$(TOP)/src/test_malloc.c \
$(TOP)/src/test_md5.c \
$(TOP)/src/test_mutex.c \
$(TOP)/src/test_onefile.c \
$(TOP)/src/test_osinst.c \
$(TOP)/src/test_pcache.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_wsd.c \
#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
TESTSRC2 = \
$(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \
$(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c \
$(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \
$(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \
$(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \
$(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \
$(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c \
$(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c \
$(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/async/sqlite3async.c
# Header files used by all library source files.
#
HDR = \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
keywordhash.h \
$(TOP)/src/mutex.h \
opcodes.h \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/pager.h \
$(TOP)/src/pcache.h \
parse.h \
sqlite3.h \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeInt.h
# Header files used by extensions
#
EXTHDR += \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_tokenizer.h
EXTHDR += \
$(TOP)/ext/rtree/rtree.h
EXTHDR += \
$(TOP)/ext/icu/sqliteicu.h
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.a sqlite3$(EXE)
libsqlite3.a: $(LIBOBJ)
$(AR) libsqlite3.a $(LIBOBJ)
$(RANLIB) libsqlite3.a
$(SHPREFIX)sqlite3.$(SO): $(LIBOBJ)
$(MKSHLIB) -o $(SHPREFIX)sqlite3.$(SO) $(LIBOBJ) $(TLIBS_SHARED)
sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \
$(TOP)/src/shell.c \
$(LIBREADLINE) $(TLIBS) $(THREADLIB) -L. -lsqlite3
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
# build on the target system. Some of the C source code and header
# files are automatically generated. This target takes care of
# all that automatic generation.
#
target_source: $(SRC)
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
touch target_source
sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl
tclsh $(TOP)/tool/mksqlite3c.tcl
cp sqlite3.c tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl
tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl
tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
# Rules to build the LEMON compiler generator
#
lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
$(BCC) -o lemon $(TOP)/tool/lemon.c
cp $(TOP)/src/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
#
# parse.o
# opcodes.o
#
%.o: %.c $(HDR)
$(TCCX_SHARED) -c $<
# Rules to build individual *.o files from files in the src directory.
#
%.o: $(TOP)/src/%.c $(HDR)
$(TCCX_SHARED) -c $<
tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR)
$(TCCX_SHARED) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | \
$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon $(OPTS) parse.y
mv parse.h parse.h.temp
awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in
sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
-e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
$(TOP)/src/sqlite.h.in >sqlite3.h
keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash >keywordhash.h
# Rules to build the extension objects.
#
icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
fts2.o: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
fts2_hash.o: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
fts2_icu.o: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
fts2_porter.o: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
fts2_tokenizer.o: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
# Rules for building test programs and for running tests
#
tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a
$(TCCX_SHARED) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
# Rules to build the 'testfixture' application.
#
TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DSQLITE_ENABLE_FTS3=1 \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fulltest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test
soaktest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test -soak=1
test: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/veryquick.test
sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
$(TOP)/tool/spaceanal.tcl
sed \
-e '/^#/d' \
-e 's,\\,\\\\,g' \
-e 's,",\\",g' \
-e 's,^,",' \
-e 's,$$,\\n",' \
$(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o sqlite3_analyzer$(EXE) \
$(LIBTCL) $(THREADLIB)
TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
./testfixture$(EXE) $(TOP)/test/loadext.test
clean:
rm -f *.o sqlite3$(EXE) libsqlite3.a sqlite3.h opcodes.*
rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
rm -f $(PUBLISH)
rm -f *.da *.bb *.bbg gmon.out
rm -rf tsrc target_source
rm -f testloadext.dll libtestloadext.so
rm -f sqlite3.c fts?amal.c tclsqlite3.c
rm -f $(SHPREFIX)sqlite3.$(SO)
+7 -11
View File
@@ -17,22 +17,18 @@ For example:
make ;# Run the makefile.
make install ;# (Optional) Install the build products
The configure script uses autoconf 2.61 and libtool. If the configure
The configure script uses autoconf 2.50 and libtool. If the configure
script does not work out for you, there is a generic makefile named
"Makefile.linux-gcc" in the top directory of the source tree that you
can copy and edit to suit your needs. Comments on the generic makefile
can copy and edit to suite your needs. Comments on the generic makefile
show what changes are needed.
The linux binaries on the website are created using the generic makefile,
not the configure script. The windows binaries on the website are created
using MinGW32 configured as a cross-compiler running under Linux. For
details, see the ./publish.sh script at the top-level of the source tree.
The developers do not use teh configure script.
SQLite does not require TCL to run, but a TCL installation is required
by the makefiles. SQLite contains a lot of generated code and TCL is
used to do much of that code generation. The makefile also requires
AWK.
not the configure script. The configure script is unmaintained. (You
can volunteer to take over maintenance of the configure script, if you want!)
The windows binaries on the website are created using MinGW32 configured
as a cross-compiler running under Linux. For details, see the ./publish.sh
script at the top-level of the source tree.
Contacts:
+1 -1
View File
@@ -1 +1 @@
3.7.6.1
3.2.8
Vendored
+3703 -5762
View File
File diff suppressed because it is too large Load Diff
-34
View File
@@ -1,34 +0,0 @@
#!/usr/bin/awk
#
# This script appends additional token codes to the end of the
# parse.h file that lemon generates. These extra token codes are
# not used by the parser. But they are used by the tokenizer and/or
# the code generator.
#
#
BEGIN {
max = 0
}
/^#define TK_/ {
print $0
if( max<$3 ) max = $3
}
END {
printf "#define TK_%-29s %4d\n", "TO_TEXT", ++max
printf "#define TK_%-29s %4d\n", "TO_BLOB", ++max
printf "#define TK_%-29s %4d\n", "TO_NUMERIC", ++max
printf "#define TK_%-29s %4d\n", "TO_INT", ++max
printf "#define TK_%-29s %4d\n", "TO_REAL", ++max
printf "#define TK_%-29s %4d\n", "ISNOT", ++max
printf "#define TK_%-29s %4d\n", "END_OF_FILE", ++max
printf "#define TK_%-29s %4d\n", "ILLEGAL", ++max
printf "#define TK_%-29s %4d\n", "SPACE", ++max
printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", ++max
printf "#define TK_%-29s %4d\n", "FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "COLUMN", ++max
printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max
printf "#define TK_%-29s %4d\n", "CONST_FUNC", ++max
printf "#define TK_%-29s %4d\n", "UMINUS", ++max
printf "#define TK_%-29s %4d\n", "UPLUS", ++max
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.3 KiB

BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.3 KiB

-5333
View File
File diff suppressed because one or more lines are too long
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 79 KiB

BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.3 KiB

+311 -414
View File
File diff suppressed because it is too large Load Diff
-104
View File
@@ -1,104 +0,0 @@
/* config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if you have the <dlfcn.h> header file. */
#undef HAVE_DLFCN_H
/* Define to 1 if you have the `fdatasync' function. */
#undef HAVE_FDATASYNC
/* Define to 1 if you have the `gmtime_r' function. */
#undef HAVE_GMTIME_R
/* Define to 1 if the system has the type `int16_t'. */
#undef HAVE_INT16_T
/* Define to 1 if the system has the type `int32_t'. */
#undef HAVE_INT32_T
/* Define to 1 if the system has the type `int64_t'. */
#undef HAVE_INT64_T
/* Define to 1 if the system has the type `int8_t'. */
#undef HAVE_INT8_T
/* Define to 1 if the system has the type `intptr_t'. */
#undef HAVE_INTPTR_T
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
/* Define to 1 if you have the `localtime_r' function. */
#undef HAVE_LOCALTIME_R
/* Define to 1 if you have the `localtime_s' function. */
#undef HAVE_LOCALTIME_S
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the <strings.h> header file. */
#undef HAVE_STRINGS_H
/* Define to 1 if you have the <string.h> header file. */
#undef HAVE_STRING_H
/* Define to 1 if you have the <sys/stat.h> header file. */
#undef HAVE_SYS_STAT_H
/* Define to 1 if you have the <sys/types.h> header file. */
#undef HAVE_SYS_TYPES_H
/* Define to 1 if the system has the type `uint16_t'. */
#undef HAVE_UINT16_T
/* Define to 1 if the system has the type `uint32_t'. */
#undef HAVE_UINT32_T
/* Define to 1 if the system has the type `uint64_t'. */
#undef HAVE_UINT64_T
/* Define to 1 if the system has the type `uint8_t'. */
#undef HAVE_UINT8_T
/* Define to 1 if the system has the type `uintptr_t'. */
#undef HAVE_UINTPTR_T
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
/* Define to 1 if you have the `usleep' function. */
#undef HAVE_USLEEP
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
#undef LT_OBJDIR
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* Define to 1 if you have the ANSI C header files. */
#undef STDC_HEADERS
/* Number of bits in a file offset, on hosts where this is settable. */
#undef _FILE_OFFSET_BITS
/* Define for large files, on AIX-style hosts. */
#undef _LARGE_FILES
Vendored
+58 -165
View File
@@ -1,10 +1,9 @@
#! /bin/sh
# Configuration validation subroutine script.
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
# 2000, 2001, 2002, 2003, 2004, 2005, 2006 Free Software Foundation,
# Inc.
# 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
timestamp='2007-06-28'
timestamp='2004-01-05'
# This file is (in principle) common to ALL GNU software.
# The presence of a machine in this file suggests that SOME GNU software
@@ -22,15 +21,14 @@ timestamp='2007-06-28'
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
# 02110-1301, USA.
#
# Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# Please send patches to <config-patches@gnu.org>. Submit a context
# diff and a properly formatted ChangeLog entry.
#
@@ -72,7 +70,7 @@ Report bugs and patches to <config-patches@gnu.org>."
version="\
GNU config.sub ($timestamp)
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
@@ -85,11 +83,11 @@ Try \`$me --help' for more information."
while test $# -gt 0 ; do
case $1 in
--time-stamp | --time* | -t )
echo "$timestamp" ; exit ;;
echo "$timestamp" ; exit 0 ;;
--version | -v )
echo "$version" ; exit ;;
echo "$version" ; exit 0 ;;
--help | --h* | -h )
echo "$usage"; exit ;;
echo "$usage"; exit 0 ;;
-- ) # Stop option processing
shift; break ;;
- ) # Use stdin as input.
@@ -101,7 +99,7 @@ while test $# -gt 0 ; do
*local*)
# First pass through any local machine types.
echo $1
exit ;;
exit 0;;
* )
break ;;
@@ -120,9 +118,8 @@ esac
# Here we must recognize all the valid KERNEL-OS combinations.
maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
case $maybe_os in
nto-qnx* | linux-gnu* | linux-dietlibc | linux-newlib* | linux-uclibc* | \
uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | \
storm-chaos* | os2-emx* | rtmk-nova*)
nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \
kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*)
os=-$maybe_os
basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
;;
@@ -148,7 +145,7 @@ case $os in
-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-apple | -axis | -knuth | -cray)
-apple | -axis)
os=
basic_machine=$1
;;
@@ -173,10 +170,6 @@ case $os in
-hiux*)
os=-hiuxwe2
;;
-sco6)
os=-sco5v6
basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
;;
-sco5)
os=-sco3.2v5
basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -193,10 +186,6 @@ case $os in
# Don't forget version if it is 3.2v4 or newer.
basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
;;
-sco5v6*)
# Don't forget version if it is 3.2v4 or newer.
basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
;;
-sco*)
os=-sco3.2v2
basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -241,16 +230,14 @@ case $basic_machine in
| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
| am33_2.0 \
| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \
| bfin \
| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
| c4x | clipper \
| d10v | d30v | dlx | dsp16xx | dvp \
| fido | fr30 | frv \
| d10v | d30v | dlx | dsp16xx \
| fr30 | frv \
| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
| i370 | i860 | i960 | ia64 \
| ip2k | iq2000 \
| m32c | m32r | m32rle | m68000 | m68k | m88k \
| maxq | mb | microblaze | mcore | mep \
| m32r | m68000 | m68k | m88k | mcore \
| mips | mipsbe | mipseb | mipsel | mipsle \
| mips16 \
| mips64 | mips64el \
@@ -259,7 +246,6 @@ case $basic_machine in
| mips64vr4100 | mips64vr4100el \
| mips64vr4300 | mips64vr4300el \
| mips64vr5000 | mips64vr5000el \
| mips64vr5900 | mips64vr5900el \
| mipsisa32 | mipsisa32el \
| mipsisa32r2 | mipsisa32r2el \
| mipsisa64 | mipsisa64el \
@@ -268,24 +254,20 @@ case $basic_machine in
| mipsisa64sr71k | mipsisa64sr71kel \
| mipstx39 | mipstx39el \
| mn10200 | mn10300 \
| mt \
| msp430 \
| nios | nios2 \
| ns16k | ns32k \
| or32 \
| openrisc | or32 \
| pdp10 | pdp11 | pj | pjl \
| powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
| pyramid \
| score \
| sh | sh[1234] | sh[24]a | sh[24]a*eb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
| sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \
| sh64 | sh64le \
| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
| spu | strongarm \
| sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \
| strongarm \
| tahoe | thumb | tic4x | tic80 | tron \
| v850 | v850e \
| we32k \
| x86 | xc16x | xscale | xscalee[bl] | xstormy16 | xtensa \
| x86 | xscale | xstormy16 | xtensa \
| z8k)
basic_machine=$basic_machine-unknown
;;
@@ -296,9 +278,6 @@ case $basic_machine in
;;
m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
;;
ms1)
basic_machine=mt-unknown
;;
# We use `pc' rather than `unknown'
# because (1) that's what they normally are, and
@@ -318,20 +297,20 @@ case $basic_machine in
| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
| avr-* | avr32-* \
| bfin-* | bs2000-* \
| avr-* \
| bs2000-* \
| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
| clipper-* | craynv-* | cydra-* \
| clipper-* | cydra-* \
| d10v-* | d30v-* | dlx-* \
| elxsi-* \
| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
| f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \
| h8300-* | h8500-* \
| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
| i*86-* | i860-* | i960-* | ia64-* \
| ip2k-* | iq2000-* \
| m32c-* | m32r-* | m32rle-* \
| m32r-* \
| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
| m88110-* | m88k-* | maxq-* | mcore-* \
| m88110-* | m88k-* | mcore-* \
| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
| mips16-* \
| mips64-* | mips64el-* \
@@ -340,7 +319,6 @@ case $basic_machine in
| mips64vr4100-* | mips64vr4100el-* \
| mips64vr4300-* | mips64vr4300el-* \
| mips64vr5000-* | mips64vr5000el-* \
| mips64vr5900-* | mips64vr5900el-* \
| mipsisa32-* | mipsisa32el-* \
| mipsisa32r2-* | mipsisa32r2el-* \
| mipsisa64-* | mipsisa64el-* \
@@ -348,28 +326,24 @@ case $basic_machine in
| mipsisa64sb1-* | mipsisa64sb1el-* \
| mipsisa64sr71k-* | mipsisa64sr71kel-* \
| mipstx39-* | mipstx39el-* \
| mmix-* \
| mt-* \
| msp430-* \
| nios-* | nios2-* \
| none-* | np1-* | ns16k-* | ns32k-* \
| none-* | np1-* | nv1-* | ns16k-* | ns32k-* \
| orion-* \
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
| pyramid-* \
| romp-* | rs6000-* \
| sh-* | sh[1234]-* | sh[24]a-* | sh[24]a*eb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
| sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \
| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
| sparclite-* \
| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | strongarm-* | sv1-* | sx?-* \
| sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \
| sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \
| tahoe-* | thumb-* \
| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
| tron-* \
| v850-* | v850e-* | vax-* \
| we32k-* \
| x86-* | x86_64-* | xc16x-* | xps100-* | xscale-* | xscalee[bl]-* \
| xstormy16-* | xtensa-* \
| x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \
| xtensa-* \
| ymp-* \
| z8k-*)
;;
@@ -389,9 +363,6 @@ case $basic_machine in
basic_machine=a29k-amd
os=-udi
;;
abacus)
basic_machine=abacus-unknown
;;
adobe68k)
basic_machine=m68010-adobe
os=-scout
@@ -471,27 +442,12 @@ case $basic_machine in
basic_machine=j90-cray
os=-unicos
;;
craynv)
basic_machine=craynv-cray
os=-unicosmp
;;
cr16)
basic_machine=cr16-unknown
os=-elf
;;
crds | unos)
basic_machine=m68k-crds
;;
crisv32 | crisv32-* | etraxfs*)
basic_machine=crisv32-axis
;;
cris | cris-* | etrax*)
basic_machine=cris-axis
;;
crx)
basic_machine=crx-unknown
os=-elf
;;
da30 | da30-*)
basic_machine=m68k-da30
;;
@@ -514,10 +470,6 @@ case $basic_machine in
basic_machine=m88k-motorola
os=-sysv3
;;
djgpp)
basic_machine=i586-pc
os=-msdosdjgpp
;;
dpx20 | dpx20-*)
basic_machine=rs6000-bull
os=-bosx
@@ -683,10 +635,6 @@ case $basic_machine in
basic_machine=i386-pc
os=-mingw32
;;
mingw32ce)
basic_machine=arm-unknown
os=-mingw32ce
;;
miniframe)
basic_machine=m68000-convergent
;;
@@ -694,30 +642,16 @@ case $basic_machine in
basic_machine=m68k-atari
os=-mint
;;
mipsEE* | ee | ps2)
basic_machine=mips64r5900el-scei
case $os in
-linux*)
;;
*)
os=-elf
;;
esac
;;
iop)
basic_machine=mipsel-scei
os=-irx
;;
dvp)
basic_machine=dvp-scei
os=-elf
;;
mips3*-*)
basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
;;
mips3*)
basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
;;
mmix*)
basic_machine=mmix-knuth
os=-mmixware
;;
monitor)
basic_machine=m68k-rom68k
os=-coff
@@ -730,9 +664,6 @@ case $basic_machine in
basic_machine=i386-pc
os=-msdos
;;
ms1-*)
basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
;;
mvs)
basic_machine=i370-ibm
os=-mvs
@@ -801,6 +732,10 @@ case $basic_machine in
np1)
basic_machine=np1-gould
;;
nv1)
basic_machine=nv1-cray
os=-unicosmp
;;
nsr-tandem)
basic_machine=nsr-tandem
;;
@@ -808,8 +743,9 @@ case $basic_machine in
basic_machine=hppa1.1-oki
os=-proelf
;;
openrisc | openrisc-*)
or32 | or32-*)
basic_machine=or32-unknown
os=-coff
;;
os400)
basic_machine=powerpc-ibm
@@ -840,12 +776,6 @@ case $basic_machine in
pc532 | pc532-*)
basic_machine=ns32k-pc532
;;
pc98)
basic_machine=i386-pc
;;
pc98-*)
basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'`
;;
pentium | p5 | k5 | k6 | nexgen | viac3)
basic_machine=i586-pc
;;
@@ -902,10 +832,6 @@ case $basic_machine in
basic_machine=i586-unknown
os=-pw32
;;
rdos)
basic_machine=i386-pc
os=-rdos
;;
rom68k)
basic_machine=m68k-rom68k
os=-coff
@@ -932,10 +858,6 @@ case $basic_machine in
sb1el)
basic_machine=mipsisa64sb1el-unknown
;;
sde)
basic_machine=mipsisa32-sde
os=-elf
;;
sei)
basic_machine=mips-sei
os=-seiux
@@ -947,9 +869,6 @@ case $basic_machine in
basic_machine=sh-hitachi
os=-hms
;;
sh5el)
basic_machine=sh5le-unknown
;;
sh64)
basic_machine=sh64-unknown
;;
@@ -1099,10 +1018,6 @@ case $basic_machine in
basic_machine=hppa1.1-winbond
os=-proelf
;;
xbox)
basic_machine=i686-pc
os=-mingw32
;;
xps | xps100)
basic_machine=xps100-honeywell
;;
@@ -1133,9 +1048,6 @@ case $basic_machine in
romp)
basic_machine=romp-ibm
;;
mmix)
basic_machine=mmix-knuth
;;
rs6000)
basic_machine=rs6000-ibm
;;
@@ -1152,10 +1064,13 @@ case $basic_machine in
we32k)
basic_machine=we32k-att
;;
sh[1234] | sh[24]a | sh[34]eb | sh[1234]le | sh[23]ele)
sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele)
basic_machine=sh-unknown
;;
sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
sh64)
basic_machine=sh64-unknown
;;
sparc | sparcv9 | sparcv9b)
basic_machine=sparc-sun
;;
cydra)
@@ -1228,23 +1143,19 @@ case $os in
| -aos* \
| -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
| -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
| -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
| -openbsd* | -solidbsd* \
| -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
| -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
| -hiux* | -386bsd* | -knetbsd* | -netbsd* | -openbsd* | -kfreebsd* | -freebsd* | -riscix* \
| -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
| -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
| -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
| -chorusos* | -chorusrdb* \
| -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
| -mingw32* | -linux-gnu* | -linux-newlib* | -linux-uclibc* \
| -uxpv* | -beos* | -mpeix* | -udk* \
| -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \
| -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
| -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
| -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
| -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
| -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
| -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -irx*)
| -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*)
# Remember, each alternative MUST END IN *, to match a version number.
;;
-qnx*)
@@ -1262,7 +1173,7 @@ case $os in
os=`echo $os | sed -e 's|nto|nto-qnx|'`
;;
-sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
| -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \
| -windows* | -osx | -abug | -netware* | -os9* | -beos* \
| -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
;;
-mac*)
@@ -1371,9 +1282,6 @@ case $os in
-kaos*)
os=-kaos
;;
-zvmoe)
os=-zvmoe
;;
-none)
;;
*)
@@ -1396,12 +1304,6 @@ else
# system, and we'll never get to this point.
case $basic_machine in
score-*)
os=-elf
;;
spu-*)
os=-elf
;;
*-acorn)
os=-riscix1.2
;;
@@ -1411,9 +1313,9 @@ case $basic_machine in
arm*-semi)
os=-aout
;;
c4x-* | tic4x-*)
os=-coff
;;
c4x-* | tic4x-*)
os=-coff
;;
# This must come before the *-dec entry.
pdp10-*)
os=-tops20
@@ -1439,9 +1341,6 @@ case $basic_machine in
m68*-cisco)
os=-aout
;;
mep-*)
os=-elf
;;
mips*-cisco)
os=-elf
;;
@@ -1460,15 +1359,9 @@ case $basic_machine in
*-be)
os=-beos
;;
*-haiku)
os=-haiku
;;
*-ibm)
os=-aix
;;
*-knuth)
os=-mmixware
;;
*-wec)
os=-proelf
;;
@@ -1634,7 +1527,7 @@ case $basic_machine in
esac
echo $basic_machine$os
exit
exit 0
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
Vendored
+16096 -10436
View File
File diff suppressed because it is too large Load Diff
+210 -295
View File
@@ -45,6 +45,18 @@
# The filename extension for executables on the build
# platform. "" for Unix and ".exe" for Windows.
#
# TARGET_CC
#
# The name of a command that runs on the build platform
# and converts C source files into *.o files for the
# target platform. In other words, the cross-compiler.
#
# TARGET_CFLAGS
#
# Switches that the target compiler needs to turn C source files
# into *.o files. Do not include TARGET_TCL_INC in this list.
# Makefiles might add additional switches such as "-I.".
#
# TCL_*
#
# Lots of values are read in from the tclConfig.sh script,
@@ -64,6 +76,18 @@
# files for the readline library. If the compiler is able
# to find <readline.h> on its own, then this can be blank.
#
# TARGET_LINK
#
# The name of the linker that combines *.o files generated
# by TARGET_CC into executables for the target platform.
#
# TARGET_LIBS
#
# Additional libraries or other switch that the target linker needs
# to build an executable on the target. Do not include
# on this list any libraries in TARGET_TCL_LIBS and
# TARGET_READLINE_LIBS, etc.
#
# TARGET_EXEEXT
#
# The filename extension for executables on the
@@ -87,78 +111,18 @@
# you don't need (for example BLT) by erasing or commenting out
# the corresponding code.
#
AC_INIT(sqlite, m4_esyscmd([cat VERSION | tr -d '\n']))
dnl Make sure the local VERSION file matches this configure script
sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
if test "$PACKAGE_VERSION" != "$sqlite_version_sanity_check" ; then
AC_MSG_ERROR([configure script is out of date:
configure \$PACKAGE_VERSION = $PACKAGE_VERSION
top level VERSION file = $sqlite_version_sanity_check
please regen with autoconf])
fi
AC_INIT(src/sqlite.h.in)
dnl Put the RCS revision string after AC_INIT so that it will also
dnl show in in configure.
# The following RCS revision string applies to configure.in
# $Revision: 1.56 $
# $Revision: 1.17 $
#########
# Programs needed
#
AC_PROG_LIBTOOL
AC_PROG_INSTALL
AC_PROG_AWK
#########
# Enable large file support (if special flags are necessary)
#
AC_SYS_LARGEFILE
#########
# Check for needed/wanted data types
AC_CHECK_TYPES([int8_t, int16_t, int32_t, int64_t, intptr_t, uint8_t,
uint16_t, uint32_t, uint64_t, uintptr_t])
#########
# Check for needed/wanted headers
AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h])
#########
# Figure out whether or not we have these functions
#
AC_CHECK_FUNCS([usleep fdatasync localtime_r gmtime_r localtime_s])
#########
# By default, we use the amalgamation (this may be changed below...)
#
USE_AMALGAMATION=1
#########
# See whether we can run specific tclsh versions known to work well;
# if not, then we fall back to plain tclsh.
# TODO: try other versions before falling back?
#
AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.5 tclsh], none)
if test "$TCLSH_CMD" = "none"; then
# If we can't find a local tclsh, then building the amalgamation will fail.
# We act as though --disable-amalgamation has been used.
echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
USE_AMALGAMATION=0
TCLSH_CMD="tclsh"
fi
AC_SUBST(TCLSH_CMD)
AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin])
if test "x${TCLLIBDIR+set}" != "xset" ; then
TCLLIBDIR='$(libdir)'
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
TCLLIBDIR=$i
break
done
TCLLIBDIR="${TCLLIBDIR}/sqlite3"
fi
#########
# Set up an appropriate program prefix
@@ -169,15 +133,15 @@ fi
AC_SUBST(program_prefix)
VERSION=[`cat $srcdir/VERSION | sed 's/^\([0-9]*\.*[0-9]*\).*/\1/'`]
AC_MSG_NOTICE(Version set to $VERSION)
echo "Version set to $VERSION"
AC_SUBST(VERSION)
RELEASE=`cat $srcdir/VERSION`
AC_MSG_NOTICE(Release set to $RELEASE)
echo "Release set to $RELEASE"
AC_SUBST(RELEASE)
VERSION_NUMBER=[`cat $srcdir/VERSION \
VERSION_NUMBER=[`cat $srcdir/VERSION \
| sed 's/[^0-9]/ /g' \
| awk '{printf "%d%03d%03d",$1,$2,$3}'`]
AC_MSG_NOTICE(Version number set to $VERSION_NUMBER)
echo "Version number set to $VERSION_NUMBER"
AC_SUBST(VERSION_NUMBER)
#########
@@ -188,7 +152,7 @@ AC_SUBST(VERSION_NUMBER)
# $system is the result of uname -s.
#
AC_ARG_WITH(hints,
AC_HELP_STRING([--with-hints=FILE],[Read configuration options from FILE]),
[ --with-hints=FILE Read configuration options from FILE],
hints=$withval)
if test "$hints" = ""; then
host=`hostname | sed 's/\..*//'`
@@ -219,66 +183,133 @@ fi
# Locate a compiler for the build machine. This compiler should
# generate command-line programs that run on the build machine.
#
if test x"$cross_compiling" = xno; then
BUILD_CC=$CC
BUILD_CFLAGS=$CFLAGS
default_build_cflags="-g"
if test "$config_BUILD_CC" = ""; then
AC_PROG_CC
if test "$cross_compiling" = "yes"; then
AC_MSG_ERROR([unable to find a compiler for building build tools])
fi
BUILD_CC=$CC
default_build_cflags=$CFLAGS
else
if test "${BUILD_CC+set}" != set; then
AC_CHECK_PROGS(BUILD_CC, gcc cc cl)
fi
if test "${BUILD_CFLAGS+set}" != set; then
BUILD_CFLAGS="-g"
fi
BUILD_CC=$config_BUILD_CC
AC_MSG_CHECKING([host compiler])
CC=$BUILD_CC
AC_MSG_RESULT($BUILD_CC)
fi
AC_MSG_CHECKING([switches for the host compiler])
if test "$config_BUILD_CFLAGS" != ""; then
CFLAGS=$config_BUILD_CFLAGS
BUILD_CFLAGS=$config_BUILD_CFLAGS
else
BUILD_CFLAGS=$default_build_cflags
fi
AC_MSG_RESULT($BUILD_CFLAGS)
if test "$config_BUILD_LIBS" != ""; then
BUILD_LIBS=$config_BUILD_LIBS
fi
AC_SUBST(BUILD_CC)
AC_SUBST(BUILD_CFLAGS)
AC_SUBST(BUILD_LIBS)
##########
# Locate a compiler that converts C code into *.o files that run on
# the target machine.
#
AC_MSG_CHECKING([target compiler])
if test "$config_TARGET_CC" != ""; then
TARGET_CC=$config_TARGET_CC
else
TARGET_CC=$BUILD_CC
fi
AC_MSG_RESULT($TARGET_CC)
AC_MSG_CHECKING([switches on the target compiler])
if test "$config_TARGET_CFLAGS" != ""; then
TARGET_CFLAGS=$config_TARGET_CFLAGS
else
TARGET_CFLAGS=$BUILD_CFLAGS
fi
AC_MSG_RESULT($TARGET_CFLAGS)
AC_MSG_CHECKING([target linker])
if test "$config_TARGET_LINK" = ""; then
TARGET_LINK=$TARGET_CC
else
TARGET_LINK=$config_TARGET_LINK
fi
AC_MSG_RESULT($TARGET_LINK)
AC_MSG_CHECKING([switches on the target compiler])
if test "$config_TARGET_TFLAGS" != ""; then
TARGET_TFLAGS=$config_TARGET_TFLAGS
else
TARGET_TFLAGS=$BUILD_CFLAGS
fi
if test "$config_TARGET_RANLIB" != ""; then
TARGET_RANLIB=$config_TARGET_RANLIB
else
AC_PROG_RANLIB
TARGET_RANLIB=$RANLIB
fi
if test "$config_TARGET_AR" != ""; then
TARGET_AR=$config_TARGET_AR
else
TARGET_AR='ar cr'
fi
AC_MSG_RESULT($TARGET_TFLAGS)
AC_SUBST(TARGET_CC)
AC_SUBST(TARGET_CFLAGS)
AC_SUBST(TARGET_LINK)
AC_SUBST(TARGET_LFLAGS)
AC_SUBST(TARGET_RANLIB)
AC_SUBST(TARGET_AR)
# Set the $cross variable if we are cross-compiling. Make
# it 0 if we are not.
#
AC_MSG_CHECKING([if host and target compilers are the same])
if test "$BUILD_CC" = "$TARGET_CC"; then
cross=0
AC_MSG_RESULT(yes)
else
cross=1
AC_MSG_RESULT(no)
fi
##########
# Do we want to support multithreaded use of sqlite
#
AC_ARG_ENABLE(threadsafe,
AC_HELP_STRING([--enable-threadsafe],[Support threadsafe operation]),,enable_threadsafe=yes)
[ --enable-threadsafe Support threadsafe operation],,enable_threadsafe=no)
AC_MSG_CHECKING([whether to support threadsafe operation])
if test "$enable_threadsafe" = "no"; then
SQLITE_THREADSAFE=0
THREADSAFE=0
AC_MSG_RESULT([no])
else
SQLITE_THREADSAFE=1
THREADSAFE=1
AC_MSG_RESULT([yes])
fi
AC_SUBST(SQLITE_THREADSAFE)
AC_SUBST(THREADSAFE)
if test "$SQLITE_THREADSAFE" = "1"; then
AC_SEARCH_LIBS(pthread_create, pthread)
fi
##########
# Do we want to allow a connection created in one thread to be used
# in another thread. This does not work on many Linux systems (ex: RedHat 9)
# due to bugs in the threading implementations. This is thus off by default.
#
AC_ARG_ENABLE(cross-thread-connections,
AC_HELP_STRING([--enable-cross-thread-connections],[Allow connection sharing across threads]),,enable_xthreadconnect=no)
AC_MSG_CHECKING([whether to allow connections to be shared across threads])
if test "$enable_xthreadconnect" = "no"; then
XTHREADCONNECT=''
AC_MSG_RESULT([no])
if test "$THREADSAFE" = "1"; then
LIBS=""
AC_CHECK_LIB(pthread, pthread_create)
TARGET_THREAD_LIB="$LIBS"
LIBS=""
else
XTHREADCONNECT='-DSQLITE_ALLOW_XTHREAD_CONNECT=1'
AC_MSG_RESULT([yes])
TARGET_THREAD_LIB=""
fi
AC_SUBST(XTHREADCONNECT)
AC_SUBST(TARGET_THREAD_LIB)
##########
# Do we want to support release
#
AC_ARG_ENABLE(releasemode,
AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
[ --enable-releasemode Support libtool link to release mode],,enable_releasemode=no)
AC_MSG_CHECKING([whether to support shared library linked as release mode or not])
if test "$enable_releasemode" = "no"; then
ALLOWRELEASE=""
AC_MSG_RESULT([no])
else
ALLOWRELEASE="-release `cat $srcdir/VERSION`"
ALLOWRELEASE="-release `cat VERSION`"
AC_MSG_RESULT([yes])
fi
AC_SUBST(ALLOWRELEASE)
@@ -287,7 +318,7 @@ AC_SUBST(ALLOWRELEASE)
# Do we want temporary databases in memory
#
AC_ARG_ENABLE(tempstore,
AC_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
[ --enable-tempstore Use an in-ram database for temporary tables (never,no,yes,always)],,enable_tempstore=yes)
AC_MSG_CHECKING([whether to use an in-ram database for temporary tables])
case "$enable_tempstore" in
never )
@@ -298,17 +329,13 @@ case "$enable_tempstore" in
TEMP_STORE=1
AC_MSG_RESULT([no])
;;
yes )
TEMP_STORE=2
AC_MSG_RESULT([yes])
;;
always )
TEMP_STORE=3
AC_MSG_RESULT([always])
;;
* )
TEMP_STORE=1
AC_MSG_RESULT([no])
TEMP_STORE=2
AC_MSG_RESULT([yes])
;;
esac
@@ -334,36 +361,38 @@ if test "$CYGWIN" = "yes"; then
else
BUILD_EXEEXT=$EXEEXT
fi
if test x"$cross_compiling" = xno; then
if test "$cross" = "0"; then
TARGET_EXEEXT=$BUILD_EXEEXT
else
TARGET_EXEEXT=$config_TARGET_EXEEXT
fi
if test "$TARGET_EXEEXT" = ".exe"; then
if test $OS2_SHELL ; then
SQLITE_OS_UNIX=0
SQLITE_OS_WIN=0
SQLITE_OS_OS2=1
CFLAGS="$CFLAGS -DSQLITE_OS_OS2=1"
else
SQLITE_OS_UNIX=0
SQLITE_OS_WIN=1
SQLITE_OS_OS2=0
CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1"
fi
OS_UNIX=0
OS_WIN=1
tclsubdir=win
TARGET_CFLAGS="$TARGET_CFLAGS -DOS_WIN=1"
else
SQLITE_OS_UNIX=1
SQLITE_OS_WIN=0
SQLITE_OS_OS2=0
CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1"
OS_UNIX=1
OS_WIN=0
tclsubdir=unix
TARGET_CFLAGS="$TARGET_CFLAGS -DOS_UNIX=1"
fi
AC_SUBST(BUILD_EXEEXT)
AC_SUBST(SQLITE_OS_UNIX)
AC_SUBST(SQLITE_OS_WIN)
AC_SUBST(SQLITE_OS_OS2)
AC_SUBST(OS_UNIX)
AC_SUBST(OS_WIN)
AC_SUBST(TARGET_EXEEXT)
##########
# Extract generic linker options from the environment.
#
if test "$config_TARGET_LIBS" != ""; then
TARGET_LIBS=$config_TARGET_LIBS
else
TARGET_LIBS=""
fi
AC_SUBST(TARGET_LIBS)
##########
# Figure out all the parameters needed to compile against Tcl.
#
@@ -372,10 +401,10 @@ AC_SUBST(TARGET_EXEEXT)
# Those macros could not be used directly since we have to make some
# minor changes to accomodate systems that do not have TCL installed.
#
AC_ARG_ENABLE(tcl, AC_HELP_STRING([--disable-tcl],[do not build TCL extension]),
AC_ARG_ENABLE(tcl, [ --disable-tcl do not build TCL extension],
[use_tcl=$enableval],[use_tcl=yes])
if test "${use_tcl}" = "yes" ; then
AC_ARG_WITH(tcl, AC_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
AC_ARG_WITH(tcl, [ --with-tcl=DIR directory containing tcl configuration (tclConfig.sh)], with_tclconfig=${withval})
AC_MSG_CHECKING([for Tcl configuration])
AC_CACHE_VAL(ac_cv_c_tclconfig,[
# First check to see if --with-tcl was specified.
@@ -386,18 +415,6 @@ if test "${use_tcl}" = "yes" ; then
AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
fi
fi
# Start autosearch by asking tclsh
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
@@ -478,7 +495,7 @@ if test "${use_tcl}" = "yes" ; then
# installed and uninstalled version of Tcl.
#
if test -f $TCL_BIN_DIR/Makefile ; then
if test -f $TCL_BIN_DIR/Makefile ; then
TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
@@ -522,179 +539,77 @@ AC_SUBST(HAVE_TCL)
# Figure out what C libraries are required to compile programs
# that use "readline()" library.
#
TARGET_READLINE_LIBS=""
TARGET_READLINE_INC=""
TARGET_HAVE_READLINE=0
AC_ARG_ENABLE([readline],
[AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])],
[with_readline=$enableval],
[with_readline=auto])
if test x"$with_readline" != xno; then
found="yes"
AC_ARG_WITH([readline-lib],
[AC_HELP_STRING([--with-readline-lib],[specify readline library])],
[with_readline_lib=$withval],
[with_readline_lib="auto"])
if test "x$with_readline_lib" = xauto; then
save_LIBS="$LIBS"
LIBS=""
AC_SEARCH_LIBS(tgetent, [readline ncurses curses termcap], [term_LIBS="$LIBS"], [term_LIBS=""])
AC_CHECK_LIB([readline], [readline], [TARGET_READLINE_LIBS="-lreadline"], [found="no"])
TARGET_READLINE_LIBS="$TARGET_READLINE_LIBS $term_LIBS"
LIBS="$save_LIBS"
else
TARGET_READLINE_LIBS="$with_readline_lib"
fi
AC_ARG_WITH([readline-inc],
[AC_HELP_STRING([--with-readline-inc],[specify readline include paths])],
[with_readline_inc=$withval],
[with_readline_inc="auto"])
if test "x$with_readline_inc" = xauto; then
AC_CHECK_HEADER(readline.h, [found="yes"], [
found="no"
if test "$cross_compiling" != yes; then
for dir in /usr /usr/local /usr/local/readline /usr/contrib /mingw; do
for subdir in include include/readline; do
AC_CHECK_FILE($dir/$subdir/readline.h, found=yes)
if test "$found" = "yes"; then
TARGET_READLINE_INC="-I$dir/$subdir"
break
fi
done
test "$found" = "yes" && break
done
fi
])
else
TARGET_READLINE_INC="$with_readline_inc"
fi
if test x"$found" = xno; then
TARGET_READLINE_LIBS=""
TARGET_READLINE_INC=""
TARGET_HAVE_READLINE=0
else
TARGET_HAVE_READLINE=1
fi
if test "$config_TARGET_READLINE_LIBS" != ""; then
TARGET_READLINE_LIBS="$config_TARGET_READLINE_LIBS"
else
CC=$TARGET_CC
LIBS=""
AC_SEARCH_LIBS(tgetent, [readline ncurses curses termcap])
AC_CHECK_LIB([readline], [readline])
TARGET_READLINE_LIBS="$LIBS"
fi
AC_SUBST(TARGET_READLINE_LIBS)
##########
# Figure out where to get the READLINE header files.
#
AC_MSG_CHECKING([readline header files])
found=no
if test "$config_TARGET_READLINE_INC" != ""; then
TARGET_READLINE_INC=$config_TARGET_READLINE_INC
found=yes
fi
if test "$found" = "yes"; then
AC_MSG_RESULT($TARGET_READLINE_INC)
else
AC_MSG_RESULT(not specified: still searching...)
AC_CHECK_HEADER(readline.h, [found=yes])
fi
if test "$found" = "no"; then
for dir in /usr /usr/local /usr/local/readline /usr/contrib /mingw; do
AC_CHECK_FILE($dir/include/readline.h, found=yes)
if test "$found" = "yes"; then
TARGET_READLINE_INC="-I$dir/include"
break
fi
AC_CHECK_FILE($dir/include/readline/readline.h, found=yes)
if test "$found" = "yes"; then
TARGET_READLINE_INC="-I$dir/include/readline"
break
fi
done
fi
if test "$found" = "yes"; then
if test "$TARGET_READLINE_LIBS" = ""; then
TARGET_HAVE_READLINE=0
else
TARGET_HAVE_READLINE=1
fi
else
TARGET_HAVE_READLINE=0
fi
AC_SUBST(TARGET_READLINE_INC)
AC_SUBST(TARGET_HAVE_READLINE)
##########
# Figure out what C libraries are required to compile programs
# that use "fdatasync()" function.
#
AC_SEARCH_LIBS(fdatasync, [rt])
#########
# check for debug enabled
AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
AC_ARG_ENABLE(debug, [ --enable-debug enable debugging & verbose explain],
[use_debug=$enableval],[use_debug=no])
if test "${use_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1"
TARGET_DEBUG=""
else
TARGET_DEBUG="-DNDEBUG"
fi
AC_SUBST(TARGET_DEBUG)
#########
# See whether we should use the amalgamation to build
AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
[Disable the amalgamation and instead build all files separately]),
[use_amalgamation=$enableval],[use_amalgamation=yes])
if test "${use_amalgamation}" != "yes" ; then
USE_AMALGAMATION=0
fi
AC_SUBST(USE_AMALGAMATION)
#########
# See whether we should allow loadable extensions
AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--enable-load-extension],
[Enable loading of external extensions]),
[use_loadextension=$enableval],[use_loadextension=no])
if test "${use_loadextension}" = "yes" ; then
OPT_FEATURE_FLAGS=""
else
OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
fi
#########
# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
for option in $CFLAGS $CPPFLAGS
do
case $option in
-DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
-DSQLITE_ENABLE*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
esac
done
AC_SUBST(OPT_FEATURE_FLAGS)
# attempt to remove any OMITS and ENABLES from the $(CFLAGS) parameter
ac_temp_CFLAGS=""
for option in $CFLAGS
do
case $option in
-DSQLITE_OMIT*) ;;
-DSQLITE_ENABLE*) ;;
*) ac_temp_CFLAGS="$ac_temp_CFLAGS $option";;
esac
done
CFLAGS=$ac_temp_CFLAGS
# attempt to remove any OMITS and ENABLES from the $(CPPFLAGS) parameter
ac_temp_CPPFLAGS=""
for option in $CPPFLAGS
do
case $option in
-DSQLITE_OMIT*) ;;
-DSQLITE_ENABLE*) ;;
*) ac_temp_CPPFLAGS="$ac_temp_CPPFLAGS $option";;
esac
done
CPPFLAGS=$ac_temp_CPPFLAGS
# attempt to remove any OMITS and ENABLES from the $(BUILD_CFLAGS) parameter
ac_temp_BUILD_CFLAGS=""
for option in $BUILD_CFLAGS
do
case $option in
-DSQLITE_OMIT*) ;;
-DSQLITE_ENABLE*) ;;
*) ac_temp_BUILD_CFLAGS="$ac_temp_BUILD_CFLAGS $option";;
esac
done
BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
#########
# See whether we should use GCOV
AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
[Enable coverage testing using gcov]),
[use_gcov=$enableval],[use_gcov=no])
if test "${use_gcov}" = "yes" ; then
USE_GCOV=1
else
USE_GCOV=0
fi
AC_SUBST(USE_GCOV)
#########
# Output the config header
AC_CONFIG_HEADERS(config.h)
# Figure out whether or not we have a "usleep()" function.
#
AC_CHECK_FUNC(usleep, [TARGET_CFLAGS="$TARGET_CFLAGS -DHAVE_USLEEP=1"])
#########
# Generate the output files.
#
AC_SUBST(BUILD_CFLAGS)
AC_OUTPUT([
Makefile
sqlite3.pc
-76
View File
@@ -1,76 +0,0 @@
*** Throughout this document, a page is deemed to have been synced
automatically as soon as it is written when PRAGMA synchronous=OFF.
Otherwise, the page is not synced until the xSync method of the VFS
is called successfully on the file containing the page.
*** Definition: A page of the database file is said to be "overwriteable" if
one or more of the following are true about the page:
(a) The original content of the page as it was at the beginning of
the transaction has been written into the rollback journal and
synced.
(b) The page was a freelist leaf page at the start of the transaction.
(c) The page number is greater than the largest page that existed in
the database file at the start of the transaction.
(1) A page of the database file is never overwritten unless one of the
following are true:
(a) The page and all other pages on the same sector are overwriteable.
(b) The atomic page write optimization is enabled, and the entire
transaction other than the update of the transaction sequence
number consists of a single page change.
(2) The content of a page written into the rollback journal exactly matches
both the content in the database when the rollback journal was written
and the content in the database at the beginning of the current
transaction.
(3) Writes to the database file are an integer multiple of the page size
in length and are aligned to a page boundary.
(4) Reads from the database file are either aligned on a page boundary and
an integer multiple of the page size in length or are taken from the
first 100 bytes of the database file.
(5) All writes to the database file are synced prior to the rollback journal
being deleted, truncated, or zeroed.
(6) If a master journal file is used, then all writes to the database file
are synced prior to the master journal being deleted.
*** Definition: Two databases (or the same database at two points it time)
are said to be "logically equivalent" if they give the same answer to
all queries. Note in particular the the content of freelist leaf
pages can be changed arbitarily without effecting the logical equivalence
of the database.
(7) At any time, if any subset, including the empty set and the total set,
of the unsynced changes to a rollback journal are removed and the
journal is rolled back, the resulting database file will be logical
equivalent to the database file at the beginning of the transaction.
(8) When a transaction is rolled back, the xTruncate method of the VFS
is called to restore the database file to the same size it was at
the beginning of the transaction. (In some VFSes, the xTruncate
method is a no-op, but that does not change the fact the SQLite will
invoke it.)
(9) Whenever the database file is modified, at least one bit in the range
of bytes from 24 through 39 inclusive will be changed prior to releasing
the EXCLUSIVE lock.
(10) The pattern of bits in bytes 24 through 39 shall not repeat in less
than one billion transactions.
(11) A database file is well-formed at the beginning and at the conclusion
of every transaction.
(12) An EXCLUSIVE lock must be held on the database file before making
any changes to the database file.
(13) A SHARED lock must be held on the database file before reading any
content out of the database file.
+121
View File
@@ -0,0 +1,121 @@
An SQLite (version 1.0) database was used in a large military application
where the database contained 105 tables and indices. The following is
a breakdown on the sizes of keys and data within these tables and indices:
Entries: 967089
Size: 45896104
Avg Size: 48
Key Size: 11112265
Avg Key Size: 12
Max Key Size: 99
0..8 263 0%
9..12 5560 0%
13..16 71394 7%
17..24 180717 26%
25..32 215442 48%
33..40 151118 64%
41..48 77479 72%
49..56 13983 74%
57..64 14481 75%
65..80 41342 79%
81..96 127098 92%
97..112 38054 96%
113..128 14197 98%
129..144 8208 99%
145..160 3326 99%
161..176 1242 99%
177..192 604 99%
193..208 222 99%
209..224 213 99%
225..240 132 99%
241..256 58 99%
257..288 515 99%
289..320 64 99%
321..352 39 99%
353..384 44 99%
385..416 25 99%
417..448 24 99%
449..480 26 99%
481..512 27 99%
513..1024 470 99%
1025..2048 396 99%
2049..4096 187 99%
4097..8192 78 99%
8193..16384 35 99%
16385..32768 17 99%
32769..65536 6 99%
65537..65541 3 100%
If the indices are omitted, the statistics for the 49 tables
become the following:
Entries: 451103
Size: 30930282
Avg Size: 69
Key Size: 1804412
Avg Key Size: 4
Max Key Size: 4
0..24 89 0%
25..32 9417 2%
33..40 119162 28%
41..48 68710 43%
49..56 9539 45%
57..64 12435 48%
65..80 38650 57%
81..96 126877 85%
97..112 38030 93%
113..128 14183 96%
129..144 7668 98%
145..160 3302 99%
161..176 1238 99%
177..192 597 99%
193..208 217 99%
209..224 211 99%
225..240 130 99%
241..256 57 99%
257..288 100 99%
289..320 62 99%
321..352 34 99%
353..384 43 99%
385..416 24 99%
417..448 24 99%
449..480 25 99%
481..512 27 99%
513..1024 153 99%
1025..2048 92 99%
2049..4096 7 100%
The 56 indices have these statistics:
Entries: 512422
Size: 14879828
Avg Size: 30
Key Size: 9253204
Avg Key Size: 19
Max Key Size: 99
0..8 246 0%
9..12 5486 1%
13..16 70717 14%
17..24 178246 49%
25..32 205722 89%
33..40 31951 96%
41..48 8768 97%
49..56 4444 98%
57..64 2046 99%
65..80 2691 99%
81..96 202 99%
97..112 11 99%
113..144 527 99%
145..160 20 99%
161..288 406 99%
289..1024 316 99%
1025..2048 304 99%
2049..4096 180 99%
4097..8192 78 99%
8193..16384 35 99%
16385..32768 17 99%
32769..65536 6 99%
65537..65541 3 100%
-130
View File
@@ -1,130 +0,0 @@
The 5 states of an historical rollback lock as implemented by the
xLock, xUnlock, and xCheckReservedLock methods of the sqlite3_io_methods
objec are:
UNLOCKED
SHARED
RESERVED
PENDING
EXCLUSIVE
The wal-index file has a similar locking hierarchy implemented using
the xShmLock method of the sqlite3_vfs object, but with 7
states. Each connection to a wal-index file must be in one of
the following 7 states:
UNLOCKED
READ
READ_FULL
WRITE
PENDING
CHECKPOINT
RECOVER
These roughly correspond to the 5 states of a rollback lock except
that SHARED is split out into 2 states: READ and READ_FULL and
there is an extra RECOVER state used for wal-index reconstruction.
The meanings of the various wal-index locking states is as follows:
UNLOCKED - The wal-index is not in use.
READ - Some prefix of the wal-index is being read. Additional
wal-index information can be appended at any time. The
newly appended content will be ignored by the holder of
the READ lock.
READ_FULL - The entire wal-index is being read. No new information
can be added to the wal-index. The holder of a READ_FULL
lock promises never to read pages from the database file
that are available anywhere in the wal-index.
WRITE - It is OK to append to the wal-index file and to adjust
the header to indicate the new "last valid frame".
PENDING - Waiting on all READ locks to clear so that a
CHECKPOINT lock can be acquired.
CHECKPOINT - It is OK to write any WAL data into the database file
and zero the last valid frame field of the wal-index
header. The wal-index file itself may not be changed
other than to zero the last valid frame field in the
header.
RECOVER - Held during wal-index recovery. Used to prevent a
race if multiple clients try to recover a wal-index at
the same time.
A particular lock manager implementation may coalesce one or more of
the wal-index locking states, though with a reduction in concurrency.
For example, an implemention might implement only exclusive locking,
in which case all states would be equivalent to CHECKPOINT, meaning that
only one reader or one writer or one checkpointer could be active at a
time. Or, an implementation might combine READ and READ_FULL into
a single state equivalent to READ, meaning that a writer could
coexist with a reader, but no reader or writers could coexist with a
checkpointer.
The lock manager must obey the following rules:
(1) A READ cannot coexist with CHECKPOINT.
(2) A READ_FULL cannot coexist with WRITE.
(3) None of WRITE, PENDING, CHECKPOINT, or RECOVER can coexist.
The SQLite core will obey the next set of rules. These rules are
assertions on the behavior of the SQLite core which might be verified
during testing using an instrumented lock manager.
(5) No part of the wal-index will be read without holding either some
kind of SHM lock or an EXCLUSIVE lock on the original database.
The original database is the file named in the 2nd parameter to
the xShmOpen method.
(6) A holder of a READ_FULL will never read any page of the database
file that is contained anywhere in the wal-index.
(7) No part of the wal-index other than the header will be written nor
will the size of the wal-index grow without holding a WRITE or
an EXCLUSIVE on the original database file.
(8) The wal-index header will not be written without holding one of
WRITE, CHECKPOINT, or RECOVER on the wal-index or an EXCLUSIVE on
the original database files.
(9) A CHECKPOINT or RECOVER must be held on the wal-index, or an
EXCLUSIVE on the original database file, in order to reset the
last valid frame counter in the header of the wal-index back to zero.
(10) A WRITE can only increase the last valid frame pointer in the header.
The SQLite core will only ever send requests for UNLOCK, READ, WRITE,
CHECKPOINT, or RECOVER to the lock manager. The SQLite core will never
request a READ_FULL or PENDING lock though the lock manager may deliver
those locking states in response to READ and CHECKPOINT requests,
respectively, if and only if the requested READ or CHECKPOINT cannot
be delivered.
The following are the allowed lock transitions:
Original-State Request New-State
-------------- ---------- ----------
(11a) UNLOCK READ READ
(11b) UNLOCK READ READ_FULL
(11c) UNLOCK CHECKPOINT PENDING
(11d) UNLOCK CHECKPOINT CHECKPOINT
(11e) READ UNLOCK UNLOCK
(11f) READ WRITE WRITE
(11g) READ RECOVER RECOVER
(11h) READ_FULL UNLOCK UNLOCK
(11i) READ_FULL WRITE WRITE
(11j) READ_FULL RECOVER RECOVER
(11k) WRITE READ READ
(11l) PENDING UNLOCK UNLOCK
(11m) PENDING CHECKPOINT CHECKPOINT
(11n) CHECKPOINT UNLOCK UNLOCK
(11o) RECOVER READ READ
These 15 transitions are all that needs to be supported. The lock
manager implementation can assert that fact. The other 27 possible
transitions among the 7 locking states will never occur.
-2
View File
@@ -1,2 +0,0 @@
Version loadable extensions to SQLite are found in subfolders
of this folder.
-164
View File
@@ -1,164 +0,0 @@
Normally, when SQLite writes to a database file, it waits until the write
operation is finished before returning control to the calling application.
Since writing to the file-system is usually very slow compared with CPU
bound operations, this can be a performance bottleneck. This directory
contains an extension that causes SQLite to perform all write requests
using a separate thread running in the background. Although this does not
reduce the overall system resources (CPU, disk bandwidth etc.) at all, it
allows SQLite to return control to the caller quickly even when writing to
the database, eliminating the bottleneck.
1. Functionality
1.1 How it Works
1.2 Limitations
1.3 Locking and Concurrency
2. Compilation and Usage
3. Porting
1. FUNCTIONALITY
With asynchronous I/O, write requests are handled by a separate thread
running in the background. This means that the thread that initiates
a database write does not have to wait for (sometimes slow) disk I/O
to occur. The write seems to happen very quickly, though in reality
it is happening at its usual slow pace in the background.
Asynchronous I/O appears to give better responsiveness, but at a price.
You lose the Durable property. With the default I/O backend of SQLite,
once a write completes, you know that the information you wrote is
safely on disk. With the asynchronous I/O, this is not the case. If
your program crashes or if a power loss occurs after the database
write but before the asynchronous write thread has completed, then the
database change might never make it to disk and the next user of the
database might not see your change.
You lose Durability with asynchronous I/O, but you still retain the
other parts of ACID: Atomic, Consistent, and Isolated. Many
appliations get along fine without the Durablity.
1.1 How it Works
Asynchronous I/O works by creating a special SQLite "vfs" structure
and registering it with sqlite3_vfs_register(). When files opened via
this vfs are written to (using the vfs xWrite() method), the data is not
written directly to disk, but is placed in the "write-queue" to be
handled by the background thread.
When files opened with the asynchronous vfs are read from
(using the vfs xRead() method), the data is read from the file on
disk and the write-queue, so that from the point of view of
the vfs reader the xWrite() appears to have already completed.
The special vfs is registered (and unregistered) by calls to the
API functions sqlite3async_initialize() and sqlite3async_shutdown().
See section "Compilation and Usage" below for details.
1.2 Limitations
In order to gain experience with the main ideas surrounding asynchronous
IO, this implementation is deliberately kept simple. Additional
capabilities may be added in the future.
For example, as currently implemented, if writes are happening at a
steady stream that exceeds the I/O capability of the background writer
thread, the queue of pending write operations will grow without bound.
If this goes on for long enough, the host system could run out of memory.
A more sophisticated module could to keep track of the quantity of
pending writes and stop accepting new write requests when the queue of
pending writes grows too large.
1.3 Locking and Concurrency
Multiple connections from within a single process that use this
implementation of asynchronous IO may access a single database
file concurrently. From the point of view of the user, if all
connections are from within a single process, there is no difference
between the concurrency offered by "normal" SQLite and SQLite
using the asynchronous backend.
If file-locking is enabled (it is enabled by default), then connections
from multiple processes may also read and write the database file.
However concurrency is reduced as follows:
* When a connection using asynchronous IO begins a database
transaction, the database is locked immediately. However the
lock is not released until after all relevant operations
in the write-queue have been flushed to disk. This means
(for example) that the database may remain locked for some
time after a "COMMIT" or "ROLLBACK" is issued.
* If an application using asynchronous IO executes transactions
in quick succession, other database users may be effectively
locked out of the database. This is because when a BEGIN
is executed, a database lock is established immediately. But
when the corresponding COMMIT or ROLLBACK occurs, the lock
is not released until the relevant part of the write-queue
has been flushed through. As a result, if a COMMIT is followed
by a BEGIN before the write-queue is flushed through, the database
is never unlocked,preventing other processes from accessing
the database.
File-locking may be disabled at runtime using the sqlite3async_control()
API (see below). This may improve performance when an NFS or other
network file-system, as the synchronous round-trips to the server be
required to establish file locks are avoided. However, if multiple
connections attempt to access the same database file when file-locking
is disabled, application crashes and database corruption is a likely
outcome.
2. COMPILATION AND USAGE
The asynchronous IO extension consists of a single file of C code
(sqlite3async.c), and a header file (sqlite3async.h) that defines the
C API used by applications to activate and control the modules
functionality.
To use the asynchronous IO extension, compile sqlite3async.c as
part of the application that uses SQLite. Then use the API defined
in sqlite3async.h to initialize and configure the module.
The asynchronous IO VFS API is described in detail in comments in
sqlite3async.h. Using the API usually consists of the following steps:
1. Register the asynchronous IO VFS with SQLite by calling the
sqlite3async_initialize() function.
2. Create a background thread to perform write operations and call
sqlite3async_run().
3. Use the normal SQLite API to read and write to databases via
the asynchronous IO VFS.
Refer to sqlite3async.h for details.
3. PORTING
Currently the asynchronous IO extension is compatible with win32 systems
and systems that support the pthreads interface, including Mac OSX, Linux,
and other varieties of Unix.
To port the asynchronous IO extension to another platform, the user must
implement mutex and condition variable primitives for the new platform.
Currently there is no externally available interface to allow this, but
modifying the code within sqlite3async.c to include the new platforms
concurrency primitives is relatively easy. Search within sqlite3async.c
for the comment string "PORTING FUNCTIONS" for details. Then implement
new versions of each of the following:
static void async_mutex_enter(int eMutex);
static void async_mutex_leave(int eMutex);
static void async_cond_wait(int eCond, int eMutex);
static void async_cond_signal(int eCond);
static void async_sched_yield(void);
The functionality required of each of the above functions is described
in comments in sqlite3async.c.
File diff suppressed because it is too large Load Diff
-223
View File
@@ -1,223 +0,0 @@
#ifndef __SQLITEASYNC_H_
#define __SQLITEASYNC_H_ 1
/*
** Make sure we can call this stuff from C++.
*/
#ifdef __cplusplus
extern "C" {
#endif
#define SQLITEASYNC_VFSNAME "sqlite3async"
/*
** THREAD SAFETY NOTES:
**
** Of the four API functions in this file, the following are not threadsafe:
**
** sqlite3async_initialize()
** sqlite3async_shutdown()
**
** Care must be taken that neither of these functions is called while
** another thread may be calling either any sqlite3async_XXX() function
** or an sqlite3_XXX() API function related to a database handle that
** is using the asynchronous IO VFS.
**
** These functions:
**
** sqlite3async_run()
** sqlite3async_control()
**
** are threadsafe. It is quite safe to call either of these functions even
** if another thread may also be calling one of them or an sqlite3_XXX()
** function related to a database handle that uses the asynchronous IO VFS.
*/
/*
** Initialize the asynchronous IO VFS and register it with SQLite using
** sqlite3_vfs_register(). If the asynchronous VFS is already initialized
** and registered, this function is a no-op. The asynchronous IO VFS
** is registered as "sqlite3async".
**
** The asynchronous IO VFS does not make operating system IO requests
** directly. Instead, it uses an existing VFS implementation for all
** required file-system operations. If the first parameter to this function
** is NULL, then the current default VFS is used for IO. If it is not
** NULL, then it must be the name of an existing VFS. In other words, the
** first argument to this function is passed to sqlite3_vfs_find() to
** locate the VFS to use for all real IO operations. This VFS is known
** as the "parent VFS".
**
** If the second parameter to this function is non-zero, then the
** asynchronous IO VFS is registered as the default VFS for all SQLite
** database connections within the process. Otherwise, the asynchronous IO
** VFS is only used by connections opened using sqlite3_open_v2() that
** specifically request VFS "sqlite3async".
**
** If a parent VFS cannot be located, then SQLITE_ERROR is returned.
** In the unlikely event that operating system specific initialization
** fails (win32 systems create the required critical section and event
** objects within this function), then SQLITE_ERROR is also returned.
** Finally, if the call to sqlite3_vfs_register() returns an error, then
** the error code is returned to the user by this function. In all three
** of these cases, intialization has failed and the asynchronous IO VFS
** is not registered with SQLite.
**
** Otherwise, if no error occurs, SQLITE_OK is returned.
*/
int sqlite3async_initialize(const char *zParent, int isDefault);
/*
** This function unregisters the asynchronous IO VFS using
** sqlite3_vfs_unregister().
**
** On win32 platforms, this function also releases the small number of
** critical section and event objects created by sqlite3async_initialize().
*/
void sqlite3async_shutdown();
/*
** This function may only be called when the asynchronous IO VFS is
** installed (after a call to sqlite3async_initialize()). It processes
** zero or more queued write operations before returning. It is expected
** (but not required) that this function will be called by a different
** thread than those threads that use SQLite. The "background thread"
** that performs IO.
**
** How many queued write operations are performed before returning
** depends on the global setting configured by passing the SQLITEASYNC_HALT
** verb to sqlite3async_control() (see below for details). By default
** this function never returns - it processes all pending operations and
** then blocks waiting for new ones.
**
** If multiple simultaneous calls are made to sqlite3async_run() from two
** or more threads, then the calls are serialized internally.
*/
void sqlite3async_run();
/*
** This function may only be called when the asynchronous IO VFS is
** installed (after a call to sqlite3async_initialize()). It is used
** to query or configure various parameters that affect the operation
** of the asynchronous IO VFS. At present there are three parameters
** supported:
**
** * The "halt" parameter, which configures the circumstances under
** which the sqlite3async_run() parameter is configured.
**
** * The "delay" parameter. Setting the delay parameter to a non-zero
** value causes the sqlite3async_run() function to sleep for the
** configured number of milliseconds between each queued write
** operation.
**
** * The "lockfiles" parameter. This parameter determines whether or
** not the asynchronous IO VFS locks the database files it operates
** on. Disabling file locking can improve throughput.
**
** This function is always passed two arguments. When setting the value
** of a parameter, the first argument must be one of SQLITEASYNC_HALT,
** SQLITEASYNC_DELAY or SQLITEASYNC_LOCKFILES. The second argument must
** be passed the new value for the parameter as type "int".
**
** When querying the current value of a paramter, the first argument must
** be one of SQLITEASYNC_GET_HALT, GET_DELAY or GET_LOCKFILES. The second
** argument to this function must be of type (int *). The current value
** of the queried parameter is copied to the memory pointed to by the
** second argument. For example:
**
** int eCurrentHalt;
** int eNewHalt = SQLITEASYNC_HALT_IDLE;
**
** sqlite3async_control(SQLITEASYNC_HALT, eNewHalt);
** sqlite3async_control(SQLITEASYNC_GET_HALT, &eCurrentHalt);
** assert( eNewHalt==eCurrentHalt );
**
** See below for more detail on each configuration parameter.
**
** SQLITEASYNC_HALT:
**
** This is used to set the value of the "halt" parameter. The second
** argument must be one of the SQLITEASYNC_HALT_XXX symbols defined
** below (either NEVER, IDLE and NOW).
**
** If the parameter is set to NEVER, then calls to sqlite3async_run()
** never return. This is the default setting. If the parameter is set
** to IDLE, then calls to sqlite3async_run() return as soon as the
** queue of pending write operations is empty. If the parameter is set
** to NOW, then calls to sqlite3async_run() return as quickly as
** possible, without processing any pending write requests.
**
** If an attempt is made to set this parameter to an integer value other
** than SQLITEASYNC_HALT_NEVER, IDLE or NOW, then sqlite3async_control()
** returns SQLITE_MISUSE and the current value of the parameter is not
** modified.
**
** Modifying the "halt" parameter affects calls to sqlite3async_run()
** made by other threads that are currently in progress.
**
** SQLITEASYNC_DELAY:
**
** This is used to set the value of the "delay" parameter. If set to
** a non-zero value, then after completing a pending write request, the
** sqlite3async_run() function sleeps for the configured number of
** milliseconds.
**
** If an attempt is made to set this parameter to a negative value,
** sqlite3async_control() returns SQLITE_MISUSE and the current value
** of the parameter is not modified.
**
** Modifying the "delay" parameter affects calls to sqlite3async_run()
** made by other threads that are currently in progress.
**
** SQLITEASYNC_LOCKFILES:
**
** This is used to set the value of the "lockfiles" parameter. This
** parameter must be set to either 0 or 1. If set to 1, then the
** asynchronous IO VFS uses the xLock() and xUnlock() methods of the
** parent VFS to lock database files being read and/or written. If
** the parameter is set to 0, then these locks are omitted.
**
** This parameter may only be set when there are no open database
** connections using the VFS and the queue of pending write requests
** is empty. Attempting to set it when this is not true, or to set it
** to a value other than 0 or 1 causes sqlite3async_control() to return
** SQLITE_MISUSE and the value of the parameter to remain unchanged.
**
** If this parameter is set to zero, then it is only safe to access the
** database via the asynchronous IO VFS from within a single process. If
** while writing to the database via the asynchronous IO VFS the database
** is also read or written from within another process, or via another
** connection that does not use the asynchronous IO VFS within the same
** process, the results are undefined (and may include crashes or database
** corruption).
**
** Alternatively, if this parameter is set to 1, then it is safe to access
** the database from multiple connections within multiple processes using
** either the asynchronous IO VFS or the parent VFS directly.
*/
int sqlite3async_control(int op, ...);
/*
** Values that can be used as the first argument to sqlite3async_control().
*/
#define SQLITEASYNC_HALT 1
#define SQLITEASYNC_GET_HALT 2
#define SQLITEASYNC_DELAY 3
#define SQLITEASYNC_GET_DELAY 4
#define SQLITEASYNC_LOCKFILES 5
#define SQLITEASYNC_GET_LOCKFILES 6
/*
** If the first argument to sqlite3async_control() is SQLITEASYNC_HALT,
** the second argument should be one of the following.
*/
#define SQLITEASYNC_HALT_NEVER 0 /* Never halt (default value) */
#define SQLITEASYNC_HALT_NOW 1 /* Halt as soon as possible */
#define SQLITEASYNC_HALT_IDLE 2 /* Halt when write-queue is empty */
#ifdef __cplusplus
} /* End of the 'extern "C"' block */
#endif
#endif /* ifndef __SQLITEASYNC_H_ */
-2
View File
@@ -1,2 +0,0 @@
This folder contains source code to the first full-text search
extension for SQLite.
-404
View File
@@ -1,404 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "ft_hash.h"
void *malloc_and_zero(int n){
void *p = malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants HASH_INT, HASH_POINTER,
** HASH_BINARY, or HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer. CopyKey only makes
** sense for HASH_STRING and HASH_BINARY and is ignored
** for other key classes.
*/
void HashInit(Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=HASH_STRING && keyClass<=HASH_BINARY );
pNew->keyClass = keyClass;
#if 0
if( keyClass==HASH_POINTER || keyClass==HASH_INT ) copyKey = 0;
#endif
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
pNew->xMalloc = malloc_and_zero;
pNew->xFree = free;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void HashClear(Hash *pH){
HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree(elem);
elem = next_elem;
}
pH->count = 0;
}
#if 0 /* NOT USED */
/*
** Hash and comparison functions when the mode is HASH_INT
*/
static int intHash(const void *pKey, int nKey){
return nKey ^ (nKey<<8) ^ (nKey>>8);
}
static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
return n2 - n1;
}
#endif
#if 0 /* NOT USED */
/*
** Hash and comparison functions when the mode is HASH_POINTER
*/
static int ptrHash(const void *pKey, int nKey){
uptr x = Addr(pKey);
return x ^ (x<<8) ^ (x>>8);
}
static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( pKey1==pKey2 ) return 0;
if( pKey1<pKey2 ) return -1;
return 1;
}
#endif
/*
** Hash and comparison functions when the mode is HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
#if 0 /* HASH_INT and HASH_POINTER are never used */
switch( keyClass ){
case HASH_INT: return &intHash;
case HASH_POINTER: return &ptrHash;
case HASH_STRING: return &strHash;
case HASH_BINARY: return &binHash;;
default: break;
}
return 0;
#else
if( keyClass==HASH_STRING ){
return &strHash;
}else{
assert( keyClass==HASH_BINARY );
return &binHash;
}
#endif
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
#if 0 /* HASH_INT and HASH_POINTER are never used */
switch( keyClass ){
case HASH_INT: return &intCompare;
case HASH_POINTER: return &ptrCompare;
case HASH_STRING: return &strCompare;
case HASH_BINARY: return &binCompare;
default: break;
}
return 0;
#else
if( keyClass==HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==HASH_BINARY );
return &binCompare;
}
#endif
}
/* Link an element into the hash table
*/
static void insertElement(
Hash *pH, /* The complete hash table */
struct _ht *pEntry, /* The entry into which pNew is inserted */
HashElem *pNew /* The element to be inserted */
){
HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(Hash *pH, int new_size){
struct _ht *new_ht; /* The new hash table */
HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
if( new_ht==0 ) return;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static HashElem *findElementGivenHash(
const Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
Hash *pH, /* The pH containing "elem" */
HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *HashFind(const Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
HashElem *elem; /* Used to loop thru the element list */
HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = pH->xMalloc( nKey );
if( new_elem->pKey==0 ){
pH->xFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
pH->xFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
-111
View File
@@ -1,111 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _HASH_H_
#define _HASH_H_
/* Forward declarations of structures. */
typedef struct Hash Hash;
typedef struct HashElem HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
HashElem *first; /* The first element of the array */
void *(*xMalloc)(int); /* malloc() function to use */
void (*xFree)(void *); /* free() function to use */
int htsize; /* Number of buckets in the hash table */
struct _ht { /* the hash table */
int count; /* Number of entries with this hash */
HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct HashElem {
HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 4 different modes of operation for a hash table:
**
** HASH_INT nKey is used as the key and pKey is ignored.
**
** HASH_POINTER pKey is used as the key and nKey is ignored.
**
** HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made for HASH_STRING and HASH_BINARY
** if the copyKey parameter to HashInit is 1.
*/
/* #define HASH_INT 1 // NOT USED */
/* #define HASH_POINTER 2 // NOT USED */
#define HASH_STRING 3
#define HASH_BINARY 4
/*
** Access routines. To delete, insert a NULL pointer.
*/
void HashInit(Hash*, int keytype, int copyKey);
void *HashInsert(Hash*, const void *pKey, int nKey, void *pData);
void *HashFind(const Hash*, const void *pKey, int nKey);
void HashClear(Hash*);
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** Hash h;
** HashElem *p;
** ...
** for(p=HashFirst(&h); p; p=HashNext(p)){
** SomeStructure *pData = HashData(p);
** // do something with pData
** }
*/
#define HashFirst(H) ((H)->first)
#define HashNext(E) ((E)->next)
#define HashData(E) ((E)->data)
#define HashKey(E) ((E)->pKey)
#define HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define HashCount(H) ((H)->count)
#endif /* _HASH_H_ */
-3345
View File
File diff suppressed because it is too large Load Diff
-11
View File
@@ -1,11 +0,0 @@
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3Fts1Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-369
View File
@@ -1,369 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include "fts1_hash.h"
static void *malloc_and_zero(int n){
void *p = malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants
** FTS1_HASH_BINARY or FTS1_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
void sqlite3Fts1HashInit(fts1Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS1_HASH_STRING && keyClass<=FTS1_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
pNew->xMalloc = malloc_and_zero;
pNew->xFree = free;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void sqlite3Fts1HashClear(fts1Hash *pH){
fts1HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
fts1HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree(elem);
elem = next_elem;
}
pH->count = 0;
}
/*
** Hash and comparison functions when the mode is FTS1_HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is FTS1_HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
if( keyClass==FTS1_HASH_STRING ){
return &strHash;
}else{
assert( keyClass==FTS1_HASH_BINARY );
return &binHash;
}
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
if( keyClass==FTS1_HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==FTS1_HASH_BINARY );
return &binCompare;
}
}
/* Link an element into the hash table
*/
static void insertElement(
fts1Hash *pH, /* The complete hash table */
struct _fts1ht *pEntry, /* The entry into which pNew is inserted */
fts1HashElem *pNew /* The element to be inserted */
){
fts1HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(fts1Hash *pH, int new_size){
struct _fts1ht *new_ht; /* The new hash table */
fts1HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts1ht *)pH->xMalloc( new_size*sizeof(struct _fts1ht) );
if( new_ht==0 ) return;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static fts1HashElem *findElementGivenHash(
const fts1Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
fts1HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts1ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
fts1Hash *pH, /* The pH containing "elem" */
fts1HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts1ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
fts1HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *sqlite3Fts1HashFind(const fts1Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
fts1HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts1HashInsert(
fts1Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
fts1HashElem *elem; /* Used to loop thru the element list */
fts1HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (fts1HashElem*)pH->xMalloc( sizeof(fts1HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = pH->xMalloc( nKey );
if( new_elem->pKey==0 ){
pH->xFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
pH->xFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
-112
View File
@@ -1,112 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _FTS1_HASH_H_
#define _FTS1_HASH_H_
/* Forward declarations of structures. */
typedef struct fts1Hash fts1Hash;
typedef struct fts1HashElem fts1HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct fts1Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
fts1HashElem *first; /* The first element of the array */
void *(*xMalloc)(int); /* malloc() function to use */
void (*xFree)(void *); /* free() function to use */
int htsize; /* Number of buckets in the hash table */
struct _fts1ht { /* the hash table */
int count; /* Number of entries with this hash */
fts1HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct fts1HashElem {
fts1HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
** FTS1_HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** FTS1_HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made if the copyKey parameter to fts1HashInit is 1.
*/
#define FTS1_HASH_STRING 1
#define FTS1_HASH_BINARY 2
/*
** Access routines. To delete, insert a NULL pointer.
*/
void sqlite3Fts1HashInit(fts1Hash*, int keytype, int copyKey);
void *sqlite3Fts1HashInsert(fts1Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts1HashFind(const fts1Hash*, const void *pKey, int nKey);
void sqlite3Fts1HashClear(fts1Hash*);
/*
** Shorthand for the functions above
*/
#define fts1HashInit sqlite3Fts1HashInit
#define fts1HashInsert sqlite3Fts1HashInsert
#define fts1HashFind sqlite3Fts1HashFind
#define fts1HashClear sqlite3Fts1HashClear
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** fts1Hash h;
** fts1HashElem *p;
** ...
** for(p=fts1HashFirst(&h); p; p=fts1HashNext(p)){
** SomeStructure *pData = fts1HashData(p);
** // do something with pData
** }
*/
#define fts1HashFirst(H) ((H)->first)
#define fts1HashNext(E) ((E)->next)
#define fts1HashData(E) ((E)->data)
#define fts1HashKey(E) ((E)->pKey)
#define fts1HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define fts1HashCount(H) ((H)->count)
#endif /* _FTS1_HASH_H_ */
-643
View File
@@ -1,643 +0,0 @@
/*
** 2006 September 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "fts1_tokenizer.h"
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
sqlite3_tokenizer base; /* Base class */
} porter_tokenizer;
/*
** Class derived from sqlit3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *zInput; /* input we are tokenizing */
int nInput; /* size of the input */
int iOffset; /* current position in zInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nAllocated; /* space allocated to zToken buffer */
} porter_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module porterTokenizerModule;
/*
** Create a new tokenizer instance.
*/
static int porterCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
porter_tokenizer *t;
t = (porter_tokenizer *) calloc(sizeof(*t), 1);
if( t==NULL ) return SQLITE_NOMEM;
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int porterDestroy(sqlite3_tokenizer *pTokenizer){
free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is zInput[0..nInput-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int porterOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, int nInput, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
porter_tokenizer_cursor *c;
c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->zInput = zInput;
if( zInput==0 ){
c->nInput = 0;
}else if( nInput<0 ){
c->nInput = (int)strlen(zInput);
}else{
c->nInput = nInput;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** porterOpen() above.
*/
static int porterClose(sqlite3_tokenizer_cursor *pCursor){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
free(c->zToken);
free(c);
return SQLITE_OK;
}
/*
** Vowel or consonant
*/
static const char cType[] = {
0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
1, 1, 1, 2, 1
};
/*
** isConsonant() and isVowel() determine if their first character in
** the string they point to is a consonant or a vowel, according
** to Porter ruls.
**
** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
** 'Y' is a consonant unless it follows another consonant,
** in which case it is a vowel.
**
** In these routine, the letters are in reverse order. So the 'y' rule
** is that 'y' is a consonant unless it is followed by another
** consonent.
*/
static int isVowel(const char*);
static int isConsonant(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return j;
return z[1]==0 || isVowel(z + 1);
}
static int isVowel(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return 1-j;
return isConsonant(z + 1);
}
/*
** Let any sequence of one or more vowels be represented by V and let
** C be sequence of one or more consonants. Then every word can be
** represented as:
**
** [C] (VC){m} [V]
**
** In prose: A word is an optional consonant followed by zero or
** vowel-consonant pairs followed by an optional vowel. "m" is the
** number of vowel consonant pairs. This routine computes the value
** of m for the first i bytes of a word.
**
** Return true if the m-value for z is 1 or more. In other words,
** return true if z contains at least one vowel that is followed
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/* Like mgt0 above except we are looking for a value of m which is
** exactly 1
*/
static int m_eq_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 1;
while( isConsonant(z) ){ z++; }
return *z==0;
}
/* Like mgt0 above except we are looking for a value of m>1 instead
** or m>0
*/
static int m_gt_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if there is a vowel anywhere within z[0..n-1]
*/
static int hasVowel(const char *z){
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if the word ends in a double consonant.
**
** The text is reversed here. So we are really looking at
** the first two characters of z[].
*/
static int doubleConsonant(const char *z){
return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
}
/*
** Return TRUE if the word ends with three letters which
** are consonant-vowel-consonent and where the final consonant
** is not 'w', 'x', or 'y'.
**
** The word is reversed here. So we are really checking the
** first three letters and the first one cannot be in [wxy].
*/
static int star_oh(const char *z){
return
z[0]!=0 && isConsonant(z) &&
z[0]!='w' && z[0]!='x' && z[0]!='y' &&
z[1]!=0 && isVowel(z+1) &&
z[2]!=0 && isConsonant(z+2);
}
/*
** If the word ends with zFrom and xCond() is true for the stem
** of the word that preceeds the zFrom ending, then change the
** ending to zTo.
**
** The input word *pz and zFrom are both in reverse order. zTo
** is in normal order.
**
** Return TRUE if zFrom matches. Return FALSE if zFrom does not
** match. Not that TRUE is returned even if xCond() fails and
** no substitution occurs.
*/
static int stem(
char **pz, /* The word being stemmed (Reversed) */
const char *zFrom, /* If the ending matches this... (Reversed) */
const char *zTo, /* ... change the ending to this (not reversed) */
int (*xCond)(const char*) /* Condition that must be true */
){
char *z = *pz;
while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
if( *zFrom!=0 ) return 0;
if( xCond && !xCond(z) ) return 1;
while( *zTo ){
*(--z) = *(zTo++);
}
*pz = z;
return 1;
}
/*
** This is the fallback stemmer used when the porter stemmer is
** inappropriate. The input word is copied into the output with
** US-ASCII case folding. If the input word is too long (more
** than 20 bytes if it contains no digits or more than 6 bytes if
** it contains digits) then word is truncated to 20 or 6 bytes
** by taking 10 or 3 bytes from the beginning and end.
*/
static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, mx, j;
int hasDigit = 0;
for(i=0; i<nIn; i++){
int c = zIn[i];
if( c>='A' && c<='Z' ){
zOut[i] = c - 'A' + 'a';
}else{
if( c>='0' && c<='9' ) hasDigit = 1;
zOut[i] = c;
}
}
mx = hasDigit ? 3 : 10;
if( nIn>mx*2 ){
for(j=mx, i=nIn-mx; i<nIn; i++, j++){
zOut[j] = zOut[i];
}
i = j;
}
zOut[i] = 0;
*pnOut = i;
}
/*
** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
** zOut is at least big enough to hold nIn bytes. Write the actual
** size of the output word (exclusive of the '\0' terminator) into *pnOut.
**
** Any upper-case characters in the US-ASCII character set ([A-Z])
** are converted to lower case. Upper-case UTF characters are
** unchanged.
**
** Words that are longer than about 20 bytes are stemmed by retaining
** a few bytes from the beginning and the end of the word. If the
** word contains digits, 3 bytes are taken from the beginning and
** 3 bytes from the end. For long words without digits, 10 bytes
** are taken from each end. US-ASCII case folding still applies.
**
** If the input word contains not digits but does characters not
** in [a-zA-Z] then no stemming is attempted and this routine just
** copies the input into the input into the output with US-ASCII
** case folding.
**
** Stemming never increases the length of the word. So there is
** no chance of overflowing the zOut buffer.
*/
static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, j, c;
char zReverse[28];
char *z, *z2;
if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
c = zIn[i];
if( c>='A' && c<='Z' ){
zReverse[j] = c + 'a' - 'A';
}else if( c>='a' && c<='z' ){
zReverse[j] = c;
}else{
/* The use of a character not in [a-zA-Z] means that we fallback
** to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
}
memset(&zReverse[sizeof(zReverse)-5], 0, 5);
z = &zReverse[j+1];
/* Step 1a */
if( z[0]=='s' ){
if(
!stem(&z, "sess", "ss", 0) &&
!stem(&z, "sei", "i", 0) &&
!stem(&z, "ss", "ss", 0)
){
z++;
}
}
/* Step 1b */
z2 = z;
if( stem(&z, "dee", "ee", m_gt_0) ){
/* Do nothing. The work was all in the test */
}else if(
(stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
&& z!=z2
){
if( stem(&z, "ta", "ate", 0) ||
stem(&z, "lb", "ble", 0) ||
stem(&z, "zi", "ize", 0) ){
/* Do nothing. The work was all in the test */
}else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
z++;
}else if( m_eq_1(z) && star_oh(z) ){
*(--z) = 'e';
}
}
/* Step 1c */
if( z[0]=='y' && hasVowel(z+1) ){
z[0] = 'i';
}
/* Step 2 */
switch( z[1] ){
case 'a':
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
break;
case 'g':
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
break;
}
/* Step 4 */
switch( z[1] ){
case 'a':
if( z[0]=='l' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'c':
if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'e':
if( z[0]=='r' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'i':
if( z[0]=='c' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'l':
if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'n':
if( z[0]=='t' ){
if( z[2]=='a' ){
if( m_gt_1(z+3) ){
z += 3;
}
}else if( z[2]=='e' ){
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
case 'o':
if( z[0]=='u' ){
if( m_gt_1(z+2) ){
z += 2;
}
}else if( z[3]=='s' || z[3]=='t' ){
stem(&z, "noi", "", m_gt_1);
}
break;
case 's':
if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
case 't':
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
z += 3;
}
break;
case 'v':
case 'z':
if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
}
/* Step 5a */
if( z[0]=='e' ){
if( m_gt_1(z+1) ){
z++;
}else if( m_eq_1(z+1) && !star_oh(z+1) ){
z++;
}
}
/* Step 5b */
if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
z++;
}
/* z[] is now the stemmed word in reverse order. Flip it back
** around into forward order and return.
*/
*pnOut = i = strlen(z);
zOut[i] = 0;
while( *z ){
zOut[--i] = *(z++);
}
}
/*
** Characters that can be part of a token. We assume any character
** whose value is greater than 0x80 (any UTF character) can be
** part of a token. In other words, delimiters all must have
** values of 0x7f or lower.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
#define idChar(C) (((ch=C)&0x80)!=0 || (ch>0x2f && isIdChar[ch-0x30]))
#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !isIdChar[ch-0x30]))
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to porterOpen().
*/
static int porterNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
const char **pzToken, /* OUT: *pzToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
const char *z = c->zInput;
while( c->iOffset<c->nInput ){
int iStartOffset, ch;
/* Scan past delimiter characters */
while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int n = c->iOffset-iStartOffset;
if( n>c->nAllocated ){
c->nAllocated = n+20;
c->zToken = realloc(c->zToken, c->nAllocated);
if( c->zToken==NULL ) return SQLITE_NOMEM;
}
porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
*pzToken = c->zToken;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the porter-stemmer tokenizer
*/
static const sqlite3_tokenizer_module porterTokenizerModule = {
0,
porterCreate,
porterDestroy,
porterOpen,
porterClose,
porterNext,
};
/*
** Allocate a new porter tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts1PorterTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &porterTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
-90
View File
@@ -1,90 +0,0 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _FTS1_TOKENIZER_H_
#define _FTS1_TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface.
*/
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
struct sqlite3_tokenizer_module {
int iVersion; /* currently 0 */
/*
** Create and destroy a tokenizer. argc/argv are passed down from
** the fulltext virtual table creation to allow customization.
*/
int (*xCreate)(int argc, const char *const*argv,
sqlite3_tokenizer **ppTokenizer);
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Tokenize a particular input. Call xOpen() to prepare to
** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
** xClose() to free any internal state. The pInput passed to
** xOpen() must exist until the cursor is closed. The ppToken
** result from xNext() is only valid until the next call to xNext()
** or until xClose() is called.
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xOpen)(sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor);
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition);
};
struct sqlite3_tokenizer {
const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
/*
** Get the module for a tokenizer which generates tokens based on a
** set of non-token characters. The default is to break tokens at any
** non-alnum character, though the set of delimiters can also be
** specified by the first argv argument to xCreate().
*/
/* TODO(shess) This doesn't belong here. Need some sort of
** registration process.
*/
void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif /* _FTS1_TOKENIZER_H_ */
-221
View File
@@ -1,221 +0,0 @@
/*
** The author disclaims copyright to this source code.
**
*************************************************************************
** Implementation of the "simple" full-text-search tokenizer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "fts1_tokenizer.h"
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
char delim[128]; /* flag ASCII delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
int iOffset; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *pToken; /* storage for current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module simpleTokenizerModule;
static int isDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) calloc(sizeof(*t), 1);
if( t==NULL ) return SQLITE_NOMEM;
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
int i, n = strlen(argv[1]);
for(i=0; i<n; i++){
unsigned char ch = argv[1][i];
/* We explicitly don't support UTF-8 delimiters for now. */
if( ch>=0x80 ){
free(t);
return SQLITE_ERROR;
}
t->delim[ch] = 1;
}
} else {
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
t->delim[i] = !isalnum(i);
}
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int simpleOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *pInput, int nBytes, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->pInput = pInput;
if( pInput==0 ){
c->nBytes = 0;
}else if( nBytes<0 ){
c->nBytes = (int)strlen(pInput);
}else{
c->nBytes = nBytes;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->pToken = NULL; /* no space allocated, yet. */
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
free(c->pToken);
free(c);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
unsigned char *p = (unsigned char *)c->pInput;
while( c->iOffset<c->nBytes ){
int iStartOffset;
/* Scan past delimiter characters */
while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int i, n = c->iOffset-iStartOffset;
if( n>c->nTokenAllocated ){
c->nTokenAllocated = n+20;
c->pToken = realloc(c->pToken, c->nTokenAllocated);
if( c->pToken==NULL ) return SQLITE_NOMEM;
}
for(i=0; i<n; i++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
/*
** Allocate a new simple tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts1SimpleTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &simpleTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
-1496
View File
File diff suppressed because it is too large Load Diff
-11
View File
@@ -1,11 +0,0 @@
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int fulltext_init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-174
View File
@@ -1,174 +0,0 @@
/*
** The author disclaims copyright to this source code.
**
*************************************************************************
** Implementation of the "simple" full-text-search tokenizer.
*/
#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "tokenizer.h"
/* Duplicate a string; the caller must free() the returned string.
* (We don't use strdup() since it's not part of the standard C library and
* may not be available everywhere.) */
/* TODO(shess) Copied from fulltext.c, consider util.c for such
** things. */
static char *string_dup(const char *s){
char *str = malloc(strlen(s) + 1);
strcpy(str, s);
return str;
}
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
const char *zDelim; /* token delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
const char *pCurrent; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nTokenBytes; /* actual size of current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
static int simpleCreate(
int argc, const char **argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
t->zDelim = string_dup(argv[1]);
} else {
/* Build a string excluding alphanumeric ASCII characters */
char zDelim[0x80]; /* nul-terminated, so nul not a member */
int i, j;
for(i=1, j=0; i<0x80; i++){
if( !isalnum(i) ){
zDelim[j++] = i;
}
}
zDelim[j++] = '\0';
assert( j<=sizeof(zDelim) );
t->zDelim = string_dup(zDelim);
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
free((void *) t->zDelim);
free(t);
return SQLITE_OK;
}
static int simpleOpen(
sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
c->pInput = pInput;
c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
c->pCurrent = c->pInput; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nTokenBytes = 0;
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
if( NULL!=c->zToken ){
free(c->zToken);
}
free(c);
return SQLITE_OK;
}
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
int ii;
while( c->pCurrent-c->pInput<c->nBytes ){
int n = (int) strcspn(c->pCurrent, t->zDelim);
if( n>0 ){
if( n+1>c->nTokenAllocated ){
c->zToken = realloc(c->zToken, n+1);
}
for(ii=0; ii<n; ii++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
char ch = c->pCurrent[ii];
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
}
c->zToken[n] = '\0';
*ppToken = c->zToken;
*pnBytes = n;
*piStartOffset = (int) (c->pCurrent-c->pInput);
*piEndOffset = *piStartOffset+n;
*piPosition = c->iToken++;
c->pCurrent += n + 1;
return SQLITE_OK;
}
c->pCurrent += n + 1;
/* TODO(shess) could strspn() to skip delimiters en masse. Needs
** to happen in two places, though, which is annoying.
*/
}
return SQLITE_DONE;
}
static sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
void get_simple_tokenizer_module(
sqlite3_tokenizer_module **ppModule
){
*ppModule = &simpleTokenizerModule;
}
-89
View File
@@ -1,89 +0,0 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _TOKENIZER_H_
#define _TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface.
*/
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
struct sqlite3_tokenizer_module {
int iVersion; /* currently 0 */
/*
** Create and destroy a tokenizer. argc/argv are passed down from
** the fulltext virtual table creation to allow customization.
*/
int (*xCreate)(int argc, const char **argv,
sqlite3_tokenizer **ppTokenizer);
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Tokenize a particular input. Call xOpen() to prepare to
** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
** xClose() to free any internal state. The pInput passed to
** xOpen() must exist until the cursor is closed. The ppToken
** result from xNext() is only valid until the next call to xNext()
** or until xClose() is called.
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xOpen)(sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor);
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition);
};
struct sqlite3_tokenizer {
sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
/*
** Get the module for a tokenizer which generates tokens based on a
** set of non-token characters. The default is to break tokens at any
** non-alnum character, though the set of delimiters can also be
** specified by the first argv argument to xCreate().
*/
/* TODO(shess) This doesn't belong here. Need some sort of
** registration process.
*/
void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
#endif /* _TOKENIZER_H_ */
-133
View File
@@ -1,133 +0,0 @@
1. FTS2 Tokenizers
When creating a new full-text table, FTS2 allows the user to select
the text tokenizer implementation to be used when indexing text
by specifying a "tokenizer" clause as part of the CREATE VIRTUAL TABLE
statement:
CREATE VIRTUAL TABLE <table-name> USING fts2(
<columns ...> [, tokenizer <tokenizer-name> [<tokenizer-args>]]
);
The built-in tokenizers (valid values to pass as <tokenizer name>) are
"simple" and "porter".
<tokenizer-args> should consist of zero or more white-space separated
arguments to pass to the selected tokenizer implementation. The
interpretation of the arguments, if any, depends on the individual
tokenizer.
2. Custom Tokenizers
FTS2 allows users to provide custom tokenizer implementations. The
interface used to create a new tokenizer is defined and described in
the fts2_tokenizer.h source file.
Registering a new FTS2 tokenizer is similar to registering a new
virtual table module with SQLite. The user passes a pointer to a
structure containing pointers to various callback functions that
make up the implementation of the new tokenizer type. For tokenizers,
the structure (defined in fts2_tokenizer.h) is called
"sqlite3_tokenizer_module".
FTS2 does not expose a C-function that users call to register new
tokenizer types with a database handle. Instead, the pointer must
be encoded as an SQL blob value and passed to FTS2 through the SQL
engine by evaluating a special scalar function, "fts2_tokenizer()".
The fts2_tokenizer() function may be called with one or two arguments,
as follows:
SELECT fts2_tokenizer(<tokenizer-name>);
SELECT fts2_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
Where <tokenizer-name> is a string identifying the tokenizer and
<sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
structure encoded as an SQL blob. If the second argument is present,
it is registered as tokenizer <tokenizer-name> and a copy of it
returned. If only one argument is passed, a pointer to the tokenizer
implementation currently registered as <tokenizer-name> is returned,
encoded as a blob. Or, if no such tokenizer exists, an SQL exception
(error) is raised.
SECURITY: If the fts2 extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they should be
prevented from invoking the fts2_tokenizer() function, possibly using the
authorisation callback.
See "Sample code" below for an example of calling the fts2_tokenizer()
function from C code.
3. ICU Library Tokenizers
If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor
symbol defined, then there exists a built-in tokenizer named "icu"
implemented using the ICU library. The first argument passed to the
xCreate() method (see fts2_tokenizer.h) of this tokenizer may be
an ICU locale identifier. For example "tr_TR" for Turkish as used
in Turkey, or "en_AU" for English as used in Australia. For example:
"CREATE VIRTUAL TABLE thai_text USING fts2(text, tokenizer icu th_TH)"
The ICU tokenizer implementation is very simple. It splits the input
text according to the ICU rules for finding word boundaries and discards
any tokens that consist entirely of white-space. This may be suitable
for some applications in some locales, but not all. If more complex
processing is required, for example to implement stemming or
discard punctuation, this can be done by creating a tokenizer
implementation that uses the ICU tokenizer as part of its implementation.
When using the ICU tokenizer this way, it is safe to overwrite the
contents of the strings returned by the xNext() method (see
fts2_tokenizer.h).
4. Sample code.
The following two code samples illustrate the way C code should invoke
the fts2_tokenizer() scalar function:
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
int queryTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
-4
View File
@@ -1,4 +0,0 @@
This folder contains source code to the second full-text search
extension for SQLite. While the API is the same, this version uses a
substantially different storage schema from fts1, so tables will need
to be rebuilt.
-6857
View File
File diff suppressed because it is too large Load Diff
-26
View File
@@ -1,26 +0,0 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This header file is used by programs that want to link against the
** FTS2 library. All it does is declare the sqlite3Fts2Init() interface.
*/
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3Fts2Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-374
View File
@@ -1,374 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "sqlite3.h"
#include "fts2_hash.h"
/*
** Malloc and Free functions
*/
static void *fts2HashMalloc(int n){
void *p = sqlite3_malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
static void fts2HashFree(void *p){
sqlite3_free(p);
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants
** FTS2_HASH_BINARY or FTS2_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
void sqlite3Fts2HashInit(fts2Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS2_HASH_STRING && keyClass<=FTS2_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void sqlite3Fts2HashClear(fts2Hash *pH){
fts2HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
fts2HashFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
fts2HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
fts2HashFree(elem->pKey);
}
fts2HashFree(elem);
elem = next_elem;
}
pH->count = 0;
}
/*
** Hash and comparison functions when the mode is FTS2_HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is FTS2_HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
if( keyClass==FTS2_HASH_STRING ){
return &strHash;
}else{
assert( keyClass==FTS2_HASH_BINARY );
return &binHash;
}
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
if( keyClass==FTS2_HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==FTS2_HASH_BINARY );
return &binCompare;
}
}
/* Link an element into the hash table
*/
static void insertElement(
fts2Hash *pH, /* The complete hash table */
struct _fts2ht *pEntry, /* The entry into which pNew is inserted */
fts2HashElem *pNew /* The element to be inserted */
){
fts2HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(fts2Hash *pH, int new_size){
struct _fts2ht *new_ht; /* The new hash table */
fts2HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts2ht *)fts2HashMalloc( new_size*sizeof(struct _fts2ht) );
if( new_ht==0 ) return;
fts2HashFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static fts2HashElem *findElementGivenHash(
const fts2Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
fts2HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts2ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
fts2Hash *pH, /* The pH containing "elem" */
fts2HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts2ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
fts2HashFree(elem->pKey);
}
fts2HashFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
fts2HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
fts2HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts2HashInsert(
fts2Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
fts2HashElem *elem; /* Used to loop thru the element list */
fts2HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (fts2HashElem*)fts2HashMalloc( sizeof(fts2HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = fts2HashMalloc( nKey );
if( new_elem->pKey==0 ){
fts2HashFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
fts2HashFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
-110
View File
@@ -1,110 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _FTS2_HASH_H_
#define _FTS2_HASH_H_
/* Forward declarations of structures. */
typedef struct fts2Hash fts2Hash;
typedef struct fts2HashElem fts2HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct fts2Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
fts2HashElem *first; /* The first element of the array */
int htsize; /* Number of buckets in the hash table */
struct _fts2ht { /* the hash table */
int count; /* Number of entries with this hash */
fts2HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct fts2HashElem {
fts2HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
** FTS2_HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** FTS2_HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made if the copyKey parameter to fts2HashInit is 1.
*/
#define FTS2_HASH_STRING 1
#define FTS2_HASH_BINARY 2
/*
** Access routines. To delete, insert a NULL pointer.
*/
void sqlite3Fts2HashInit(fts2Hash*, int keytype, int copyKey);
void *sqlite3Fts2HashInsert(fts2Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts2HashFind(const fts2Hash*, const void *pKey, int nKey);
void sqlite3Fts2HashClear(fts2Hash*);
/*
** Shorthand for the functions above
*/
#define fts2HashInit sqlite3Fts2HashInit
#define fts2HashInsert sqlite3Fts2HashInsert
#define fts2HashFind sqlite3Fts2HashFind
#define fts2HashClear sqlite3Fts2HashClear
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** fts2Hash h;
** fts2HashElem *p;
** ...
** for(p=fts2HashFirst(&h); p; p=fts2HashNext(p)){
** SomeStructure *pData = fts2HashData(p);
** // do something with pData
** }
*/
#define fts2HashFirst(H) ((H)->first)
#define fts2HashNext(E) ((E)->next)
#define fts2HashData(E) ((E)->data)
#define fts2HashKey(E) ((E)->pKey)
#define fts2HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define fts2HashCount(H) ((H)->count)
#endif /* _FTS2_HASH_H_ */
-260
View File
@@ -1,260 +0,0 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements a tokenizer for fts2 based on the ICU library.
**
** $Id: fts2_icu.c,v 1.3 2008/12/18 05:30:26 danielk1977 Exp $
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#ifdef SQLITE_ENABLE_ICU
#include <assert.h>
#include <string.h>
#include "fts2_tokenizer.h"
#include <unicode/ubrk.h>
#include <unicode/ucol.h>
#include <unicode/ustring.h>
#include <unicode/utf16.h>
typedef struct IcuTokenizer IcuTokenizer;
typedef struct IcuCursor IcuCursor;
struct IcuTokenizer {
sqlite3_tokenizer base;
char *zLocale;
};
struct IcuCursor {
sqlite3_tokenizer_cursor base;
UBreakIterator *pIter; /* ICU break-iterator object */
int nChar; /* Number of UChar elements in pInput */
UChar *aChar; /* Copy of input using utf-16 encoding */
int *aOffset; /* Offsets of each character in utf-8 input */
int nBuffer;
char *zBuffer;
int iToken;
};
/*
** Create a new tokenizer instance.
*/
static int icuCreate(
int argc, /* Number of entries in argv[] */
const char * const *argv, /* Tokenizer creation arguments */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
){
IcuTokenizer *p;
int n = 0;
if( argc>0 ){
n = strlen(argv[0])+1;
}
p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
if( !p ){
return SQLITE_NOMEM;
}
memset(p, 0, sizeof(IcuTokenizer));
if( n ){
p->zLocale = (char *)&p[1];
memcpy(p->zLocale, argv[0], n);
}
*ppTokenizer = (sqlite3_tokenizer *)p;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int icuDestroy(sqlite3_tokenizer *pTokenizer){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
sqlite3_free(p);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int icuOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, /* Input string */
int nInput, /* Length of zInput in bytes */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
IcuCursor *pCsr;
const int32_t opt = U_FOLD_CASE_DEFAULT;
UErrorCode status = U_ZERO_ERROR;
int nChar;
UChar32 c;
int iInput = 0;
int iOut = 0;
*ppCursor = 0;
if( nInput<0 ){
nInput = strlen(zInput);
}
nChar = nInput+1;
pCsr = (IcuCursor *)sqlite3_malloc(
sizeof(IcuCursor) + /* IcuCursor */
nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
(nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
);
if( !pCsr ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(IcuCursor));
pCsr->aChar = (UChar *)&pCsr[1];
pCsr->aOffset = (int *)&pCsr->aChar[nChar];
pCsr->aOffset[iOut] = iInput;
U8_NEXT(zInput, iInput, nInput, c);
while( c>0 ){
int isError = 0;
c = u_foldCase(c, opt);
U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
if( isError ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->aOffset[iOut] = iInput;
if( iInput<nInput ){
U8_NEXT(zInput, iInput, nInput, c);
}else{
c = 0;
}
}
pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
if( !U_SUCCESS(status) ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->nChar = iOut;
ubrk_first(pCsr->pIter);
*ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to icuOpen().
*/
static int icuClose(sqlite3_tokenizer_cursor *pCursor){
IcuCursor *pCsr = (IcuCursor *)pCursor;
ubrk_close(pCsr->pIter);
sqlite3_free(pCsr->zBuffer);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor.
*/
static int icuNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
IcuCursor *pCsr = (IcuCursor *)pCursor;
int iStart = 0;
int iEnd = 0;
int nByte = 0;
while( iStart==iEnd ){
UChar32 c;
iStart = ubrk_current(pCsr->pIter);
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
}
do {
UErrorCode status = U_ZERO_ERROR;
if( nByte ){
char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
if( !zNew ){
return SQLITE_NOMEM;
}
pCsr->zBuffer = zNew;
pCsr->nBuffer = nByte;
}
u_strToUTF8(
pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
&pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
&status /* Output success/failure */
);
} while( nByte>pCsr->nBuffer );
*ppToken = pCsr->zBuffer;
*pnBytes = nByte;
*piStartOffset = pCsr->aOffset[iStart];
*piEndOffset = pCsr->aOffset[iEnd];
*piPosition = pCsr->iToken++;
return SQLITE_OK;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
};
/*
** Set *ppModule to point at the implementation of the ICU tokenizer.
*/
void sqlite3Fts2IcuTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &icuTokenizerModule;
}
#endif /* defined(SQLITE_ENABLE_ICU) */
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
-641
View File
@@ -1,641 +0,0 @@
/*
** 2006 September 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fts2_tokenizer.h"
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
sqlite3_tokenizer base; /* Base class */
} porter_tokenizer;
/*
** Class derived from sqlit3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *zInput; /* input we are tokenizing */
int nInput; /* size of the input */
int iOffset; /* current position in zInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nAllocated; /* space allocated to zToken buffer */
} porter_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module porterTokenizerModule;
/*
** Create a new tokenizer instance.
*/
static int porterCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
porter_tokenizer *t;
t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int porterDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is zInput[0..nInput-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int porterOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, int nInput, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
porter_tokenizer_cursor *c;
c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->zInput = zInput;
if( zInput==0 ){
c->nInput = 0;
}else if( nInput<0 ){
c->nInput = (int)strlen(zInput);
}else{
c->nInput = nInput;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** porterOpen() above.
*/
static int porterClose(sqlite3_tokenizer_cursor *pCursor){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
sqlite3_free(c->zToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Vowel or consonant
*/
static const char cType[] = {
0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
1, 1, 1, 2, 1
};
/*
** isConsonant() and isVowel() determine if their first character in
** the string they point to is a consonant or a vowel, according
** to Porter ruls.
**
** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
** 'Y' is a consonant unless it follows another consonant,
** in which case it is a vowel.
**
** In these routine, the letters are in reverse order. So the 'y' rule
** is that 'y' is a consonant unless it is followed by another
** consonent.
*/
static int isVowel(const char*);
static int isConsonant(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return j;
return z[1]==0 || isVowel(z + 1);
}
static int isVowel(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return 1-j;
return isConsonant(z + 1);
}
/*
** Let any sequence of one or more vowels be represented by V and let
** C be sequence of one or more consonants. Then every word can be
** represented as:
**
** [C] (VC){m} [V]
**
** In prose: A word is an optional consonant followed by zero or
** vowel-consonant pairs followed by an optional vowel. "m" is the
** number of vowel consonant pairs. This routine computes the value
** of m for the first i bytes of a word.
**
** Return true if the m-value for z is 1 or more. In other words,
** return true if z contains at least one vowel that is followed
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/* Like mgt0 above except we are looking for a value of m which is
** exactly 1
*/
static int m_eq_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 1;
while( isConsonant(z) ){ z++; }
return *z==0;
}
/* Like mgt0 above except we are looking for a value of m>1 instead
** or m>0
*/
static int m_gt_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if there is a vowel anywhere within z[0..n-1]
*/
static int hasVowel(const char *z){
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if the word ends in a double consonant.
**
** The text is reversed here. So we are really looking at
** the first two characters of z[].
*/
static int doubleConsonant(const char *z){
return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
}
/*
** Return TRUE if the word ends with three letters which
** are consonant-vowel-consonent and where the final consonant
** is not 'w', 'x', or 'y'.
**
** The word is reversed here. So we are really checking the
** first three letters and the first one cannot be in [wxy].
*/
static int star_oh(const char *z){
return
z[0]!=0 && isConsonant(z) &&
z[0]!='w' && z[0]!='x' && z[0]!='y' &&
z[1]!=0 && isVowel(z+1) &&
z[2]!=0 && isConsonant(z+2);
}
/*
** If the word ends with zFrom and xCond() is true for the stem
** of the word that preceeds the zFrom ending, then change the
** ending to zTo.
**
** The input word *pz and zFrom are both in reverse order. zTo
** is in normal order.
**
** Return TRUE if zFrom matches. Return FALSE if zFrom does not
** match. Not that TRUE is returned even if xCond() fails and
** no substitution occurs.
*/
static int stem(
char **pz, /* The word being stemmed (Reversed) */
const char *zFrom, /* If the ending matches this... (Reversed) */
const char *zTo, /* ... change the ending to this (not reversed) */
int (*xCond)(const char*) /* Condition that must be true */
){
char *z = *pz;
while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
if( *zFrom!=0 ) return 0;
if( xCond && !xCond(z) ) return 1;
while( *zTo ){
*(--z) = *(zTo++);
}
*pz = z;
return 1;
}
/*
** This is the fallback stemmer used when the porter stemmer is
** inappropriate. The input word is copied into the output with
** US-ASCII case folding. If the input word is too long (more
** than 20 bytes if it contains no digits or more than 6 bytes if
** it contains digits) then word is truncated to 20 or 6 bytes
** by taking 10 or 3 bytes from the beginning and end.
*/
static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, mx, j;
int hasDigit = 0;
for(i=0; i<nIn; i++){
int c = zIn[i];
if( c>='A' && c<='Z' ){
zOut[i] = c - 'A' + 'a';
}else{
if( c>='0' && c<='9' ) hasDigit = 1;
zOut[i] = c;
}
}
mx = hasDigit ? 3 : 10;
if( nIn>mx*2 ){
for(j=mx, i=nIn-mx; i<nIn; i++, j++){
zOut[j] = zOut[i];
}
i = j;
}
zOut[i] = 0;
*pnOut = i;
}
/*
** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
** zOut is at least big enough to hold nIn bytes. Write the actual
** size of the output word (exclusive of the '\0' terminator) into *pnOut.
**
** Any upper-case characters in the US-ASCII character set ([A-Z])
** are converted to lower case. Upper-case UTF characters are
** unchanged.
**
** Words that are longer than about 20 bytes are stemmed by retaining
** a few bytes from the beginning and the end of the word. If the
** word contains digits, 3 bytes are taken from the beginning and
** 3 bytes from the end. For long words without digits, 10 bytes
** are taken from each end. US-ASCII case folding still applies.
**
** If the input word contains not digits but does characters not
** in [a-zA-Z] then no stemming is attempted and this routine just
** copies the input into the input into the output with US-ASCII
** case folding.
**
** Stemming never increases the length of the word. So there is
** no chance of overflowing the zOut buffer.
*/
static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, j, c;
char zReverse[28];
char *z, *z2;
if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
c = zIn[i];
if( c>='A' && c<='Z' ){
zReverse[j] = c + 'a' - 'A';
}else if( c>='a' && c<='z' ){
zReverse[j] = c;
}else{
/* The use of a character not in [a-zA-Z] means that we fallback
** to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
}
memset(&zReverse[sizeof(zReverse)-5], 0, 5);
z = &zReverse[j+1];
/* Step 1a */
if( z[0]=='s' ){
if(
!stem(&z, "sess", "ss", 0) &&
!stem(&z, "sei", "i", 0) &&
!stem(&z, "ss", "ss", 0)
){
z++;
}
}
/* Step 1b */
z2 = z;
if( stem(&z, "dee", "ee", m_gt_0) ){
/* Do nothing. The work was all in the test */
}else if(
(stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
&& z!=z2
){
if( stem(&z, "ta", "ate", 0) ||
stem(&z, "lb", "ble", 0) ||
stem(&z, "zi", "ize", 0) ){
/* Do nothing. The work was all in the test */
}else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
z++;
}else if( m_eq_1(z) && star_oh(z) ){
*(--z) = 'e';
}
}
/* Step 1c */
if( z[0]=='y' && hasVowel(z+1) ){
z[0] = 'i';
}
/* Step 2 */
switch( z[1] ){
case 'a':
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
break;
case 'g':
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
break;
}
/* Step 4 */
switch( z[1] ){
case 'a':
if( z[0]=='l' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'c':
if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'e':
if( z[0]=='r' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'i':
if( z[0]=='c' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'l':
if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'n':
if( z[0]=='t' ){
if( z[2]=='a' ){
if( m_gt_1(z+3) ){
z += 3;
}
}else if( z[2]=='e' ){
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
case 'o':
if( z[0]=='u' ){
if( m_gt_1(z+2) ){
z += 2;
}
}else if( z[3]=='s' || z[3]=='t' ){
stem(&z, "noi", "", m_gt_1);
}
break;
case 's':
if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
case 't':
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
z += 3;
}
break;
case 'v':
case 'z':
if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
}
/* Step 5a */
if( z[0]=='e' ){
if( m_gt_1(z+1) ){
z++;
}else if( m_eq_1(z+1) && !star_oh(z+1) ){
z++;
}
}
/* Step 5b */
if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
z++;
}
/* z[] is now the stemmed word in reverse order. Flip it back
** around into forward order and return.
*/
*pnOut = i = strlen(z);
zOut[i] = 0;
while( *z ){
zOut[--i] = *(z++);
}
}
/*
** Characters that can be part of a token. We assume any character
** whose value is greater than 0x80 (any UTF character) can be
** part of a token. In other words, delimiters all must have
** values of 0x7f or lower.
*/
static const char porterIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to porterOpen().
*/
static int porterNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
const char **pzToken, /* OUT: *pzToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
const char *z = c->zInput;
while( c->iOffset<c->nInput ){
int iStartOffset, ch;
/* Scan past delimiter characters */
while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int n = c->iOffset-iStartOffset;
if( n>c->nAllocated ){
c->nAllocated = n+20;
c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
if( c->zToken==NULL ) return SQLITE_NOMEM;
}
porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
*pzToken = c->zToken;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the porter-stemmer tokenizer
*/
static const sqlite3_tokenizer_module porterTokenizerModule = {
0,
porterCreate,
porterDestroy,
porterOpen,
porterClose,
porterNext,
};
/*
** Allocate a new porter tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts2PorterTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &porterTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
-371
View File
@@ -1,371 +0,0 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is part of an SQLite module implementing full-text search.
** This particular file implements the generic tokenizer interface.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include "fts2_hash.h"
#include "fts2_tokenizer.h"
#include <assert.h>
/*
** Implementation of the SQL scalar function for accessing the underlying
** hash table. This function may be called as follows:
**
** SELECT <function-name>(<key-name>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
**
** If the <pointer> argument is specified, it must be a blob value
** containing a pointer to be stored as the hash data corresponding
** to the string <key-name>. If <pointer> is not specified, then
** the string <key-name> must already exist in the has table. Otherwise,
** an error is returned.
**
** Whether or not the <pointer> argument is specified, the value returned
** is a blob containing the pointer stored as the hash data corresponding
** to string <key-name> (after the hash-table is updated, if applicable).
*/
static void scalarFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
fts2Hash *pHash;
void *pPtr = 0;
const unsigned char *zName;
int nName;
assert( argc==1 || argc==2 );
pHash = (fts2Hash *)sqlite3_user_data(context);
zName = sqlite3_value_text(argv[0]);
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( n!=sizeof(pPtr) ){
sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
pPtr = *(void **)sqlite3_value_blob(argv[1]);
pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
if( pOld==pPtr ){
sqlite3_result_error(context, "out of memory", -1);
return;
}
}else{
pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
if( !pPtr ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
}
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
#ifdef SQLITE_TEST
#include <tcl.h>
#include <string.h>
/*
** Implementation of a special SQL scalar function for testing tokenizers
** designed to be used in concert with the Tcl testing framework. This
** function must be called with two arguments:
**
** SELECT <function-name>(<key-name>, <input-string>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
**
** The return value is a string that may be interpreted as a Tcl
** list. For each token in the <input-string>, three elements are
** added to the returned list. The first is the token position, the
** second is the token text (folded, stemmed, etc.) and the third is the
** substring of <input-string> associated with the token. For example,
** using the built-in "simple" tokenizer:
**
** SELECT fts_tokenizer_test('simple', 'I don't see how');
**
** will return the string:
**
** "{0 i I 1 dont don't 2 see see 3 how how}"
**
*/
static void testFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
fts2Hash *pHash;
sqlite3_tokenizer_module *p;
sqlite3_tokenizer *pTokenizer = 0;
sqlite3_tokenizer_cursor *pCsr = 0;
const char *zErr = 0;
const char *zName;
int nName;
const char *zInput;
int nInput;
const char *zArg = 0;
const char *zToken;
int nToken;
int iStart;
int iEnd;
int iPos;
Tcl_Obj *pRet;
assert( argc==2 || argc==3 );
nName = sqlite3_value_bytes(argv[0]);
zName = (const char *)sqlite3_value_text(argv[0]);
nInput = sqlite3_value_bytes(argv[argc-1]);
zInput = (const char *)sqlite3_value_text(argv[argc-1]);
if( argc==3 ){
zArg = (const char *)sqlite3_value_text(argv[1]);
}
pHash = (fts2Hash *)sqlite3_user_data(context);
p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
if( !p ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
zErr = "error in xCreate()";
goto finish;
}
pTokenizer->pModule = p;
if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
zErr = "error in xOpen()";
goto finish;
}
pCsr->pTokenizer = pTokenizer;
while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
zToken = &zInput[iStart];
nToken = iEnd-iStart;
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
}
if( SQLITE_OK!=p->xClose(pCsr) ){
zErr = "error in xClose()";
goto finish;
}
if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
zErr = "error in xDestroy()";
goto finish;
}
finish:
if( zErr ){
sqlite3_result_error(context, zErr, -1);
}else{
sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
}
Tcl_DecrRefCount(pRet);
}
static
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
static
int queryFts2Tokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
/*
** Implementation of the scalar function fts2_tokenizer_internal_test().
** This function is used for testing only, it is not included in the
** build unless SQLITE_TEST is defined.
**
** The purpose of this is to test that the fts2_tokenizer() function
** can be used as designed by the C-code in the queryFts2Tokenizer and
** registerTokenizer() functions above. These two functions are repeated
** in the README.tokenizer file as an example, so it is important to
** test them.
**
** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
** function with no arguments. An assert() will fail if a problem is
** detected. i.e.:
**
** SELECT fts2_tokenizer_internal_test();
**
*/
static void intTestFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int rc;
const sqlite3_tokenizer_module *p1;
const sqlite3_tokenizer_module *p2;
sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
/* Test the query function */
sqlite3Fts2SimpleTokenizerModule(&p1);
rc = queryFts2Tokenizer(db, "simple", &p2);
assert( rc==SQLITE_OK );
assert( p1==p2 );
rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_ERROR );
assert( p2==0 );
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
#endif
/*
** Set up SQL objects in database db used to access the contents of
** the hash table pointed to by argument pHash. The hash table must
** been initialised to use string keys, and to take a private copy
** of the key when a value is inserted. i.e. by a call similar to:
**
** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
**
** This function adds a scalar function (see header comment above
** scalarFunc() in this file for details) and, if ENABLE_TABLE is
** defined at compilation time, a temporary virtual table (see header
** comment above struct HashTableVtab) to the database schema. Both
** provide read/write access to the contents of *pHash.
**
** The third argument to this function, zName, is used as the name
** of both the scalar and, if created, the virtual table.
*/
int sqlite3Fts2InitHashTable(
sqlite3 *db,
fts2Hash *pHash,
const char *zName
){
int rc = SQLITE_OK;
void *p = (void *)pHash;
const int any = SQLITE_ANY;
char *zTest = 0;
char *zTest2 = 0;
#ifdef SQLITE_TEST
void *pdb = (void *)db;
zTest = sqlite3_mprintf("%s_test", zName);
zTest2 = sqlite3_mprintf("%s_internal_test", zName);
if( !zTest || !zTest2 ){
rc = SQLITE_NOMEM;
}
#endif
if( rc!=SQLITE_OK
|| (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
#ifdef SQLITE_TEST
|| (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
#endif
);
sqlite3_free(zTest);
sqlite3_free(zTest2);
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
-145
View File
@@ -1,145 +0,0 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _FTS2_TOKENIZER_H_
#define _FTS2_TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface. When a new tokenizer
** implementation is registered, the caller provides a pointer to
** an sqlite3_tokenizer_module containing pointers to the callback
** functions that make up an implementation.
**
** When an fts2 table is created, it passes any arguments passed to
** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
** implementation. The xCreate() function in turn returns an
** sqlite3_tokenizer structure representing the specific tokenizer to
** be used for the fts2 table (customized by the tokenizer clause arguments).
**
** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
** method is called. It returns an sqlite3_tokenizer_cursor object
** that may be used to tokenize a specific input buffer based on
** the tokenization rules supplied by a specific sqlite3_tokenizer
** object.
*/
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
struct sqlite3_tokenizer_module {
/*
** Structure version. Should always be set to 0.
*/
int iVersion;
/*
** Create a new tokenizer. The values in the argv[] array are the
** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
** TABLE statement that created the fts2 table. For example, if
** the following SQL is executed:
**
** CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2)
**
** then argc is set to 2, and the argv[] array contains pointers
** to the strings "arg1" and "arg2".
**
** This method should return either SQLITE_OK (0), or an SQLite error
** code. If SQLITE_OK is returned, then *ppTokenizer should be set
** to point at the newly created tokenizer structure. The generic
** sqlite3_tokenizer.pModule variable should not be initialised by
** this callback. The caller will do so.
*/
int (*xCreate)(
int argc, /* Size of argv array */
const char *const*argv, /* Tokenizer argument strings */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
);
/*
** Destroy an existing tokenizer. The fts2 module calls this method
** exactly once for each successful call to xCreate().
*/
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Create a tokenizer cursor to tokenize an input buffer. The caller
** is responsible for ensuring that the input buffer remains valid
** until the cursor is closed (using the xClose() method).
*/
int (*xOpen)(
sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
const char *pInput, int nBytes, /* Input buffer */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
);
/*
** Destroy an existing tokenizer cursor. The fts2 module calls this
** method exactly once for each successful call to xOpen().
*/
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
/*
** Retrieve the next token from the tokenizer cursor pCursor. This
** method should either return SQLITE_OK and set the values of the
** "OUT" variables identified below, or SQLITE_DONE to indicate that
** the end of the buffer has been reached, or an SQLite error code.
**
** *ppToken should be set to point at a buffer containing the
** normalized version of the token (i.e. after any case-folding and/or
** stemming has been performed). *pnBytes should be set to the length
** of this buffer in bytes. The input text that generated the token is
** identified by the byte offsets returned in *piStartOffset and
** *piEndOffset.
**
** The buffer *ppToken is set to point at is managed by the tokenizer
** implementation. It is only required to be valid until the next call
** to xNext() or xClose().
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xNext)(
sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
int *piStartOffset, /* OUT: Byte offset of token in input buffer */
int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
int *piPosition /* OUT: Number of tokens returned before this one */
);
};
struct sqlite3_tokenizer {
const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
#endif /* _FTS2_TOKENIZER_H_ */
-230
View File
@@ -1,230 +0,0 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Implementation of the "simple" full-text-search tokenizer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fts2_tokenizer.h"
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
char delim[128]; /* flag ASCII delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
int iOffset; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *pToken; /* storage for current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module simpleTokenizerModule;
static int simpleDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
int i, n = strlen(argv[1]);
for(i=0; i<n; i++){
unsigned char ch = argv[1][i];
/* We explicitly don't support UTF-8 delimiters for now. */
if( ch>=0x80 ){
sqlite3_free(t);
return SQLITE_ERROR;
}
t->delim[ch] = 1;
}
} else {
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
(i>='a' && i<='z'));
}
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int simpleOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *pInput, int nBytes, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->pInput = pInput;
if( pInput==0 ){
c->nBytes = 0;
}else if( nBytes<0 ){
c->nBytes = (int)strlen(pInput);
}else{
c->nBytes = nBytes;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->pToken = NULL; /* no space allocated, yet. */
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
sqlite3_free(c->pToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
unsigned char *p = (unsigned char *)c->pInput;
while( c->iOffset<c->nBytes ){
int iStartOffset;
/* Scan past delimiter characters */
while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int i, n = c->iOffset-iStartOffset;
if( n>c->nTokenAllocated ){
c->nTokenAllocated = n+20;
c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
if( c->pToken==NULL ) return SQLITE_NOMEM;
}
for(i=0; i<n; i++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
/*
** Allocate a new simple tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts2SimpleTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &simpleTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
-116
View File
@@ -1,116 +0,0 @@
#!/usr/bin/tclsh
#
# This script builds a single C code file holding all of FTS2 code.
# The name of the output file is fts2amal.c. To build this file,
# first do:
#
# make target_source
#
# The make target above moves all of the source code files into
# a subdirectory named "tsrc". (This script expects to find the files
# there and will not work if they are not found.)
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
# tclsh mkfts2amal.tcl
#
# The amalgamated FTS2 code will be written into fts2amal.c
#
# Open the output file and write a header comment at the beginning
# of the file.
#
set out [open fts2amal.c w]
set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
puts $out [subst \
{/******************************************************************************
** This file is an amalgamation of separate C source files from the SQLite
** Full Text Search extension 2 (fts2). By combining all the individual C
** code files into this single large file, the entire code can be compiled
** as a one translation unit. This allows many compilers to do optimizations
** that would not be possible if the files were compiled separately. It also
** makes the code easier to import into other projects.
**
** This amalgamation was generated on $today.
*/}]
# These are the header files used by FTS2. The first time any of these
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line. The file only needs to be included once.
#
foreach hdr {
fts2.h
fts2_hash.h
fts2_tokenizer.h
sqlite3.h
sqlite3ext.h
} {
set available_hdr($hdr) 1
}
# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}
# Insert a comment into the code
#
proc section_comment {text} {
global out s78
set n [string length $text]
set nstar [expr {60 - $n}]
set stars [string range $s78 0 $nstar]
puts $out "/************** $text $stars/"
}
# Read the source file named $filename and write it into the
# sqlite3.c output file. If any #include statements are seen,
# process them approprately.
#
proc copy_file {filename} {
global seen_hdr available_hdr out
set tail [file tail $filename]
section_comment "Begin file $tail"
set in [open $filename r]
while {![eof $in]} {
set line [gets $in]
if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
if {[info exists available_hdr($hdr)]} {
if {$available_hdr($hdr)} {
section_comment "Include $hdr in the middle of $tail"
copy_file tsrc/$hdr
section_comment "Continuing where we left off in $tail"
}
} elseif {![info exists seen_hdr($hdr)]} {
set seen_hdr($hdr) 1
puts $out $line
}
} elseif {[regexp {^#ifdef __cplusplus} $line]} {
puts $out "#if 0"
} elseif {[regexp {^#line} $line]} {
# Skip #line directives.
} else {
puts $out $line
}
}
close $in
section_comment "End of $tail"
}
# Process the source files. Process files containing commonly
# used subroutines first in order to help the compiler find
# inlining opportunities.
#
foreach file {
fts2.c
fts2_hash.c
fts2_porter.c
fts2_tokenizer.c
fts2_tokenizer1.c
fts2_icu.c
} {
copy_file tsrc/$file
}
close $out
-209
View File
@@ -1,209 +0,0 @@
1. OVERVIEW
This README file describes the syntax of the arguments that may be passed to
the FTS3 MATCH operator used for full-text queries. For example, if table
"t1" is an Fts3 virtual table, the following SQL query:
SELECT * FROM t1 WHERE <col> MATCH <full-text query>
may be used to retrieve all rows that match a specified for full-text query.
The text "<col>" should be replaced by either the name of the fts3 table
(in this case "t1"), or by the name of one of the columns of the fts3
table. <full-text-query> should be replaced by an SQL expression that
computes to a string containing an Fts3 query.
If the left-hand-side of the MATCH operator is set to the name of the
fts3 table, then by default the query may be matched against any column
of the table. If it is set to a column name, then by default the query
may only match the specified column. In both cases this may be overriden
as part of the query text (see sections 2 and 3 below).
As of SQLite version 3.6.8, Fts3 supports two slightly different query
formats; the standard syntax, which is used by default, and the enhanced
query syntax which can be selected by compiling with the pre-processor
symbol SQLITE_ENABLE_FTS3_PARENTHESIS defined.
-DSQLITE_ENABLE_FTS3_PARENTHESIS
2. STANDARD QUERY SYNTAX
When using the standard Fts3 query syntax, a query usually consists of a
list of terms (words) separated by white-space characters. To match a
query, a row (or column) of an Fts3 table must contain each of the specified
terms. For example, the following query:
<col> MATCH 'hello world'
matches rows (or columns, if <col> is the name of a column name) that
contain at least one instance of the token "hello", and at least one
instance of the token "world". Tokens may be grouped into phrases using
quotation marks. In this case, a matching row or column must contain each
of the tokens in the phrase in the order specified, with no intervening
tokens. For example, the query:
<col> MATCH '"hello world" joe"
matches the first of the following two documents, but not the second or
third:
"'Hello world', said Joe."
"One should always greet the world with a cheery hello, thought Joe."
"How many hello world programs could their be?"
As well as grouping tokens together by phrase, the binary NEAR operator
may be used to search for rows that contain two or more specified tokens
or phrases within a specified proximity of each other. The NEAR operator
must always be specified in upper case. The word "near" in lower or mixed
case is treated as an ordinary token. For example, the following query:
<col> MATCH 'engineering NEAR consultancy'
matches rows that contain both the "engineering" and "consultancy" tokens
in the same column with not more than 10 other words between them. It does
not matter which of the two terms occurs first in the document, only that
they be seperated by only 10 tokens or less. The user may also specify
a different required proximity by adding "/N" immediately after the NEAR
operator, where N is an integer. For example:
<col> MATCH 'engineering NEAR/5 consultancy'
searches for a row containing an instance of each specified token seperated
by not more than 5 other tokens. More than one NEAR operator can be used
in as sequence. For example this query:
<col> MATCH 'reliable NEAR/2 engineering NEAR/5 consultancy'
searches for a row that contains an instance of the token "reliable"
seperated by not more than two tokens from an instance of "engineering",
which is in turn separated by not more than 5 other tokens from an
instance of the term "consultancy". Phrases enclosed in quotes may
also be used as arguments to the NEAR operator.
Similar to the NEAR operator, one or more tokens or phrases may be
separated by OR operators. In this case, only one of the specified tokens
or phrases must appear in the document. For example, the query:
<col> MATCH 'hello OR world'
matches rows that contain either the term "hello", or the term "world",
or both. Note that unlike in many programming languages, the OR operator
has a higher precedence than the AND operators implied between white-space
separated tokens. The following query matches documents that contain the
term 'sqlite' and at least one of the terms 'fantastic' or 'impressive',
not those that contain both 'sqlite' and 'fantastic' or 'impressive':
<col> MATCH 'sqlite fantastic OR impressive'
Any token that is part of an Fts3 query expression, whether or not it is
part of a phrase enclosed in quotes, may have a '*' character appended to
it. In this case, the token matches all terms that begin with the characters
of the token, not just those that exactly match it. For example, the
following query:
<col> MATCH 'sql*'
matches all rows that contain the term "SQLite", as well as those that
contain "SQL".
A token that is not part of a quoted phrase may be preceded by a '-'
character, which indicates that matching rows must not contain the
specified term. For example, the following:
<col> MATCH '"database engine" -sqlite'
matches rows that contain the phrase "database engine" but do not contain
the term "sqlite". If the '-' character occurs inside a quoted phrase,
it is ignored. It is possible to use both the '-' prefix and the '*' postfix
on a single term. At this time, all Fts3 queries must contain at least
one term or phrase that is not preceded by the '-' prefix.
Regardless of whether or not a table name or column name is used on the
left hand side of the MATCH operator, a specific column of the fts3 table
may be associated with each token in a query by preceding a token with
a column name followed by a ':' character. For example, regardless of what
is specified for <col>, the following query requires that column "col1"
of the table contains the term "hello", and that column "col2" of the
table contains the term "world". If the table does not contain columns
named "col1" and "col2", then an error is returned and the query is
not run.
<col> MATCH 'col1:hello col2:world'
It is not possible to associate a specific table column with a quoted
phrase or a term preceded by a '-' operator. A '*' character may be
appended to a term associated with a specific column for prefix matching.
3. ENHANCED QUERY SYNTAX
The enhanced query syntax is quite similar to the standard query syntax,
with the following four differences:
1) Parenthesis are supported. When using the enhanced query syntax,
parenthesis may be used to overcome the built-in precedence of the
supplied binary operators. For example, the following query:
<col> MATCH '(hello world) OR (simple example)'
matches documents that contain both "hello" and "world", and documents
that contain both "simple" and "example". It is not possible to forumlate
such a query using the standard syntax.
2) Instead of separating tokens and phrases by whitespace, an AND operator
may be explicitly specified. This does not change query processing at
all, but may be used to improve readability. For example, the following
query is handled identically to the one above:
<col> MATCH '(hello AND world) OR (simple AND example)'
As with the OR and NEAR operators, the AND operator must be specified
in upper case. The word "and" specified in lower or mixed case is
handled as a regular token.
3) The '-' token prefix is not supported. Instead, a new binary operator,
NOT, is included. The NOT operator requires that the query specified
as its left-hand operator matches, but that the query specified as the
right-hand operator does not. For example, to query for all rows that
contain the term "example" but not the term "simple", the following
query could be used:
<col> MATCH 'example NOT simple'
As for all other operators, the NOT operator must be specified in
upper case. Otherwise it will be treated as a regular token.
4) Unlike in the standard syntax, where the OR operator has a higher
precedence than the implicit AND operator, when using the enhanced
syntax implicit and explict AND operators have a higher precedence
than OR operators. Using the enhanced syntax, the following two
queries are equivalent:
<col> MATCH 'sqlite fantastic OR impressive'
<col> MATCH '(sqlite AND fantastic) OR impressive'
however, when using the standard syntax, the query:
<col> MATCH 'sqlite fantastic OR impressive'
is equivalent to the enhanced syntax query:
<col> MATCH 'sqlite AND (fantastic OR impressive)'
The precedence of all enhanced syntax operators, in order from highest
to lowest, is:
NEAR (highest precedence, tightest grouping)
NOT
AND
OR (lowest precedence, loosest grouping)
Using the advanced syntax, it is possible to specify expressions enclosed
in parenthesis as operands to the NOT, AND and OR operators. However both
the left and right hand side operands of NEAR operators must be either
tokens or phrases. Attempting the following query will return an error:
<col> MATCH 'sqlite NEAR (fantastic OR impressive)'
Queries of this form must be re-written as:
<col> MATCH 'sqlite NEAR fantastic OR sqlite NEAR impressive'
-133
View File
@@ -1,133 +0,0 @@
1. FTS3 Tokenizers
When creating a new full-text table, FTS3 allows the user to select
the text tokenizer implementation to be used when indexing text
by specifying a "tokenize" clause as part of the CREATE VIRTUAL TABLE
statement:
CREATE VIRTUAL TABLE <table-name> USING fts3(
<columns ...> [, tokenize <tokenizer-name> [<tokenizer-args>]]
);
The built-in tokenizers (valid values to pass as <tokenizer name>) are
"simple" and "porter".
<tokenizer-args> should consist of zero or more white-space separated
arguments to pass to the selected tokenizer implementation. The
interpretation of the arguments, if any, depends on the individual
tokenizer.
2. Custom Tokenizers
FTS3 allows users to provide custom tokenizer implementations. The
interface used to create a new tokenizer is defined and described in
the fts3_tokenizer.h source file.
Registering a new FTS3 tokenizer is similar to registering a new
virtual table module with SQLite. The user passes a pointer to a
structure containing pointers to various callback functions that
make up the implementation of the new tokenizer type. For tokenizers,
the structure (defined in fts3_tokenizer.h) is called
"sqlite3_tokenizer_module".
FTS3 does not expose a C-function that users call to register new
tokenizer types with a database handle. Instead, the pointer must
be encoded as an SQL blob value and passed to FTS3 through the SQL
engine by evaluating a special scalar function, "fts3_tokenizer()".
The fts3_tokenizer() function may be called with one or two arguments,
as follows:
SELECT fts3_tokenizer(<tokenizer-name>);
SELECT fts3_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
Where <tokenizer-name> is a string identifying the tokenizer and
<sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
structure encoded as an SQL blob. If the second argument is present,
it is registered as tokenizer <tokenizer-name> and a copy of it
returned. If only one argument is passed, a pointer to the tokenizer
implementation currently registered as <tokenizer-name> is returned,
encoded as a blob. Or, if no such tokenizer exists, an SQL exception
(error) is raised.
SECURITY: If the fts3 extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they should be
prevented from invoking the fts3_tokenizer() function, possibly using the
authorisation callback.
See "Sample code" below for an example of calling the fts3_tokenizer()
function from C code.
3. ICU Library Tokenizers
If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor
symbol defined, then there exists a built-in tokenizer named "icu"
implemented using the ICU library. The first argument passed to the
xCreate() method (see fts3_tokenizer.h) of this tokenizer may be
an ICU locale identifier. For example "tr_TR" for Turkish as used
in Turkey, or "en_AU" for English as used in Australia. For example:
"CREATE VIRTUAL TABLE thai_text USING fts3(text, tokenizer icu th_TH)"
The ICU tokenizer implementation is very simple. It splits the input
text according to the ICU rules for finding word boundaries and discards
any tokens that consist entirely of white-space. This may be suitable
for some applications in some locales, but not all. If more complex
processing is required, for example to implement stemming or
discard punctuation, this can be done by creating a tokenizer
implementation that uses the ICU tokenizer as part of its implementation.
When using the ICU tokenizer this way, it is safe to overwrite the
contents of the strings returned by the xNext() method (see
fts3_tokenizer.h).
4. Sample code.
The following two code samples illustrate the way C code should invoke
the fts3_tokenizer() scalar function:
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
int queryTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
-4
View File
@@ -1,4 +0,0 @@
This folder contains source code to the second full-text search
extension for SQLite. While the API is the same, this version uses a
substantially different storage schema from fts1, so tables will need
to be rebuilt.
-3813
View File
File diff suppressed because it is too large Load Diff
-26
View File
@@ -1,26 +0,0 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This header file is used by programs that want to link against the
** FTS3 library. All it does is declare the sqlite3Fts3Init() interface.
*/
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3Fts3Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-389
View File
@@ -1,389 +0,0 @@
/*
** 2009 Nov 12
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#ifndef _FTSINT_H
#define _FTSINT_H
#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
# define NDEBUG 1
#endif
#include "sqlite3.h"
#include "fts3_tokenizer.h"
#include "fts3_hash.h"
/*
** This constant controls how often segments are merged. Once there are
** FTS3_MERGE_COUNT segments of level N, they are merged into a single
** segment of level N+1.
*/
#define FTS3_MERGE_COUNT 16
/*
** This is the maximum amount of data (in bytes) to store in the
** Fts3Table.pendingTerms hash table. Normally, the hash table is
** populated as documents are inserted/updated/deleted in a transaction
** and used to create a new segment when the transaction is committed.
** However if this limit is reached midway through a transaction, a new
** segment is created and the hash table cleared immediately.
*/
#define FTS3_MAX_PENDING_DATA (1*1024*1024)
/*
** Macro to return the number of elements in an array. SQLite has a
** similar macro called ArraySize(). Use a different name to avoid
** a collision when building an amalgamation with built-in FTS3.
*/
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
/*
** Maximum length of a varint encoded integer. The varint format is different
** from that used by SQLite, so the maximum length is 10, not 9.
*/
#define FTS3_VARINT_MAX 10
/*
** The testcase() macro is only used by the amalgamation. If undefined,
** make it a no-op.
*/
#ifndef testcase
# define testcase(X)
#endif
/*
** Terminator values for position-lists and column-lists.
*/
#define POS_COLUMN (1) /* Column-list terminator */
#define POS_END (0) /* Position-list terminator */
/*
** This section provides definitions to allow the
** FTS3 extension to be compiled outside of the
** amalgamation.
*/
#ifndef SQLITE_AMALGAMATION
/*
** Macros indicating that conditional expressions are always true or
** false.
*/
#ifdef SQLITE_COVERAGE_TEST
# define ALWAYS(x) (1)
# define NEVER(X) (0)
#else
# define ALWAYS(x) (x)
# define NEVER(X) (x)
#endif
/*
** Internal types used by SQLite.
*/
typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
typedef short int i16; /* 2-byte (or larger) signed integer */
typedef unsigned int u32; /* 4-byte unsigned integer */
typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
/*
** Macro used to suppress compiler warnings for unused parameters.
*/
#define UNUSED_PARAMETER(x) (void)(x)
#endif
typedef struct Fts3Table Fts3Table;
typedef struct Fts3Cursor Fts3Cursor;
typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3PhraseToken Fts3PhraseToken;
typedef struct Fts3SegFilter Fts3SegFilter;
typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
** of this structure and xDestroy and xDisconnect free that instance.
** All other methods receive a pointer to the structure as one of their
** arguments.
*/
struct Fts3Table {
sqlite3_vtab base; /* Base class used by SQLite core */
sqlite3 *db; /* The database connection */
const char *zDb; /* logical database name */
const char *zName; /* virtual table name */
int nColumn; /* number of named columns in virtual table */
char **azColumn; /* column names. malloced */
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[24];
char *zReadExprlist;
char *zWriteExprlist;
int nNodeSize; /* Soft limit for node size */
u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
/* The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
** pending data, including hash table overhead, but not malloc overhead.
** When nPendingData exceeds nMaxPendingData, the buffer is flushed
** automatically. Variable iPrevDocid is the docid of the most recently
** inserted record.
*/
int nMaxPendingData;
int nPendingData;
sqlite_int64 iPrevDocid;
Fts3Hash pendingTerms;
};
/*
** When the core wants to read from the virtual table, it creates a
** virtual table cursor (an instance of the following structure) using
** the xOpen method. Cursors are destroyed using the xClose method.
*/
struct Fts3Cursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
i16 eSearch; /* Search strategy (see below) */
u8 isEof; /* True if at End Of Results */
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
int nPhrase; /* Number of matchable phrases in query */
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
char *pNextId; /* Pointer into the body of aDoclist */
char *aDoclist; /* List of docids for full-text queries */
int nDoclist; /* Size of buffer at aDoclist */
int desc; /* True to sort in descending order */
int eEvalmode; /* An FTS3_EVAL_XX constant */
int nRowAvg; /* Average size of database rows, in pages */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
int nMatchinfo; /* Number of elements in aMatchinfo[] */
char *zMatchinfo; /* Matchinfo specification */
};
#define FTS3_EVAL_FILTER 0
#define FTS3_EVAL_NEXT 1
#define FTS3_EVAL_MATCHINFO 2
/*
** The Fts3Cursor.eSearch member is always set to one of the following.
** Actualy, Fts3Cursor.eSearch can be greater than or equal to
** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
** of the column to be searched. For example, in
**
** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
**
** Because the LHS of the MATCH operator is 2nd column "b",
** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
** indicating that all columns should be searched,
** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
*/
#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case.
** For a sequence of tokens contained in double-quotes (i.e. "one two three")
** nToken will be the number of tokens in the string.
**
** The nDocMatch and nMatch variables contain data that may be used by the
** matchinfo() function. They are populated when the full-text index is
** queried for hits on the phrase. If one or more tokens in the phrase
** are deferred, the nDocMatch and nMatch variables are populated based
** on the assumption that the
*/
struct Fts3PhraseToken {
char *z; /* Text of the token */
int n; /* Number of bytes in buffer z */
int isPrefix; /* True if token ends with a "*" character */
int bFulltext; /* True if full-text index was used */
Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
};
struct Fts3Phrase {
/* Variables populated by fts3_expr.c when parsing a MATCH expression */
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
int isNot; /* Phrase prefixed by unary not (-) operator */
Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};
/*
** A tree of these objects forms the RHS of a MATCH operator.
**
** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
** is true, then aDoclist points to a malloced buffer, size nDoclist bytes,
** containing the results of the NEAR or phrase query in FTS3 doclist
** format. As usual, the initial "Length" field found in doclists stored
** on disk is omitted from this buffer.
**
** Variable pCurrent always points to the start of a docid field within
** aDoclist. Since the doclist is usually scanned in docid order, this can
** be used to accelerate seeking to the required docid within the doclist.
*/
struct Fts3Expr {
int eType; /* One of the FTSQUERY_XXX values defined below */
int nNear; /* Valid if eType==FTSQUERY_NEAR */
Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
Fts3Expr *pLeft; /* Left operand */
Fts3Expr *pRight; /* Right operand */
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
int isLoaded; /* True if aDoclist/nDoclist are initialized. */
char *aDoclist; /* Buffer containing doclist */
int nDoclist; /* Size of aDoclist in bytes */
sqlite3_int64 iCurrent;
char *pCurrent;
};
/*
** Candidate values for Fts3Query.eType. Note that the order of the first
** four values is in order of precedence when parsing expressions. For
** example, the following:
**
** "a OR b AND c NOT d NEAR e"
**
** is equivalent to:
**
** "a OR (b AND (c NOT (d NEAR e)))"
*/
#define FTSQUERY_NEAR 1
#define FTSQUERY_NOT 2
#define FTSQUERY_AND 3
#define FTSQUERY_OR 4
#define FTSQUERY_PHRASE 5
/* fts3_write.c */
int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);
int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
void sqlite3Fts3SegmentsClose(Fts3Table *);
#define FTS3_SEGCURSOR_PENDING -1
#define FTS3_SEGCURSOR_ALL -2
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
int sqlite3Fts3SegReaderCursor(
Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
#define FTS3_SEGMENT_PREFIX 0x00000008
#define FTS3_SEGMENT_SCAN 0x00000010
/* Type passed as 4th argument to SegmentReaderIterate() */
struct Fts3SegFilter {
const char *zTerm;
int nTerm;
int iCol;
int flags;
};
struct Fts3SegReaderCursor {
/* Used internally by sqlite3Fts3SegReaderXXX() calls */
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
int nSegment; /* Size of apSegment array */
int nAdvance; /* How many seg-readers to advance */
Fts3SegFilter *pFilter; /* Pointer to filter object */
char *aBuffer; /* Buffer to merge doclists in */
int nBuffer; /* Allocated size of aBuffer[] in bytes */
/* Cost of running this iterator. Used by fts3.c only. */
int nCost;
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
char *zTerm; /* Pointer to term buffer */
int nTerm; /* Size of zTerm in bytes */
char *aDoclist; /* Pointer to doclist buffer */
int nDoclist; /* Size of aDoclist[] in bytes */
};
/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
sqlite3_tokenizer **, char **
);
int sqlite3Fts3IsIdChar(char);
/* fts3_snippet.c */
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif
/* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db);
#endif /* _FTSINT_H */
-470
View File
@@ -1,470 +0,0 @@
/*
** 2011 Jan 27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include "fts3Int.h"
#include <string.h>
#include <assert.h>
typedef struct Fts3auxTable Fts3auxTable;
typedef struct Fts3auxCursor Fts3auxCursor;
struct Fts3auxTable {
sqlite3_vtab base; /* Base class used by SQLite core */
Fts3Table *pFts3Tab;
};
struct Fts3auxCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3SegFilter filter;
char *zStop;
int nStop; /* Byte-length of string zStop */
int isEof; /* True if cursor is at EOF */
sqlite3_int64 iRowid; /* Current rowid */
int iCol; /* Current value of 'col' column */
int nStat; /* Size of aStat[] array */
struct Fts3auxColstats {
sqlite3_int64 nDoc; /* 'documents' values for current csr row */
sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */
} *aStat;
};
/*
** Schema of the terms table.
*/
#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)"
/*
** This function does all the work for both the xConnect and xCreate methods.
** These tables have no persistent representation of their own, so xConnect
** and xCreate are identical operations.
*/
static int fts3auxConnectMethod(
sqlite3 *db, /* Database connection */
void *pUnused, /* Unused */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
char const *zDb; /* Name of database (e.g. "main") */
char const *zFts3; /* Name of fts3 table */
int nDb; /* Result of strlen(zDb) */
int nFts3; /* Result of strlen(zFts3) */
int nByte; /* Bytes of space to allocate here */
int rc; /* value returned by declare_vtab() */
Fts3auxTable *p; /* Virtual table object to return */
UNUSED_PARAMETER(pUnused);
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
*pzErr = sqlite3_mprintf(
"wrong number of arguments to fts4aux constructor"
);
return SQLITE_ERROR;
}
zDb = argv[1];
nDb = strlen(zDb);
zFts3 = argv[3];
nFts3 = strlen(zFts3);
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
p = (Fts3auxTable *)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, nByte);
p->pFts3Tab = (Fts3Table *)&p[1];
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
*ppVtab = (sqlite3_vtab *)p;
return SQLITE_OK;
}
/*
** This function does the work for both the xDisconnect and xDestroy methods.
** These tables have no persistent representation of their own, so xDisconnect
** and xDestroy are identical operations.
*/
static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){
Fts3auxTable *p = (Fts3auxTable *)pVtab;
Fts3Table *pFts3 = p->pFts3Tab;
int i;
/* Free any prepared statements held */
for(i=0; i<SizeofArray(pFts3->aStmt); i++){
sqlite3_finalize(pFts3->aStmt[i]);
}
sqlite3_free(pFts3->zSegmentsTbl);
sqlite3_free(p);
return SQLITE_OK;
}
#define FTS4AUX_EQ_CONSTRAINT 1
#define FTS4AUX_GE_CONSTRAINT 2
#define FTS4AUX_LE_CONSTRAINT 4
/*
** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts3auxBestIndexMethod(
sqlite3_vtab *pVTab,
sqlite3_index_info *pInfo
){
int i;
int iEq = -1;
int iGe = -1;
int iLe = -1;
UNUSED_PARAMETER(pVTab);
/* This vtab delivers always results in "ORDER BY term ASC" order. */
if( pInfo->nOrderBy==1
&& pInfo->aOrderBy[0].iColumn==0
&& pInfo->aOrderBy[0].desc==0
){
pInfo->orderByConsumed = 1;
}
/* Search for equality and range constraints on the "term" column. */
for(i=0; i<pInfo->nConstraint; i++){
if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){
int op = pInfo->aConstraint[i].op;
if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
}
}
if( iEq>=0 ){
pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT;
pInfo->aConstraintUsage[iEq].argvIndex = 1;
pInfo->estimatedCost = 5;
}else{
pInfo->idxNum = 0;
pInfo->estimatedCost = 20000;
if( iGe>=0 ){
pInfo->idxNum += FTS4AUX_GE_CONSTRAINT;
pInfo->aConstraintUsage[iGe].argvIndex = 1;
pInfo->estimatedCost /= 2;
}
if( iLe>=0 ){
pInfo->idxNum += FTS4AUX_LE_CONSTRAINT;
pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0);
pInfo->estimatedCost /= 2;
}
}
return SQLITE_OK;
}
/*
** xOpen - Open a cursor.
*/
static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts3auxCursor *pCsr; /* Pointer to cursor object to return */
UNUSED_PARAMETER(pVTab);
pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor));
if( !pCsr ) return SQLITE_NOMEM;
memset(pCsr, 0, sizeof(Fts3auxCursor));
*ppCsr = (sqlite3_vtab_cursor *)pCsr;
return SQLITE_OK;
}
/*
** xClose - Close a cursor.
*/
static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
sqlite3Fts3SegmentsClose(pFts3);
sqlite3Fts3SegReaderFinish(&pCsr->csr);
sqlite3_free((void *)pCsr->filter.zTerm);
sqlite3_free(pCsr->zStop);
sqlite3_free(pCsr->aStat);
sqlite3_free(pCsr);
return SQLITE_OK;
}
static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
if( nSize>pCsr->nStat ){
struct Fts3auxColstats *aNew;
aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat,
sizeof(struct Fts3auxColstats) * nSize
);
if( aNew==0 ) return SQLITE_NOMEM;
memset(&aNew[pCsr->nStat], 0,
sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat)
);
pCsr->aStat = aNew;
pCsr->nStat = nSize;
}
return SQLITE_OK;
}
/*
** xNext - Advance the cursor to the next row, if any.
*/
static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
int rc;
/* Increment our pretend rowid value. */
pCsr->iRowid++;
for(pCsr->iCol++; pCsr->iCol<pCsr->nStat; pCsr->iCol++){
if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK;
}
rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
if( rc==SQLITE_ROW ){
int i = 0;
int nDoclist = pCsr->csr.nDoclist;
char *aDoclist = pCsr->csr.aDoclist;
int iCol;
int eState = 0;
if( pCsr->zStop ){
int n = (pCsr->nStop<pCsr->csr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm;
int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n);
if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){
pCsr->isEof = 1;
return SQLITE_OK;
}
}
if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM;
memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat);
iCol = 0;
while( i<nDoclist ){
sqlite3_int64 v = 0;
i += sqlite3Fts3GetVarint(&aDoclist[i], &v);
switch( eState ){
/* State 0. In this state the integer just read was a docid. */
case 0:
pCsr->aStat[0].nDoc++;
eState = 1;
iCol = 0;
break;
/* State 1. In this state we are expecting either a 1, indicating
** that the following integer will be a column number, or the
** start of a position list for column 0.
**
** The only difference between state 1 and state 2 is that if the
** integer encountered in state 1 is not 0 or 1, then we need to
** increment the column 0 "nDoc" count for this term.
*/
case 1:
assert( iCol==0 );
if( v>1 ){
pCsr->aStat[1].nDoc++;
}
eState = 2;
/* fall through */
case 2:
if( v==0 ){ /* 0x00. Next integer will be a docid. */
eState = 0;
}else if( v==1 ){ /* 0x01. Next integer will be a column number. */
eState = 3;
}else{ /* 2 or greater. A position. */
pCsr->aStat[iCol+1].nOcc++;
pCsr->aStat[0].nOcc++;
}
break;
/* State 3. The integer just read is a column number. */
default: assert( eState==3 );
iCol = (int)v;
if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM;
pCsr->aStat[iCol+1].nDoc++;
eState = 2;
break;
}
}
pCsr->iCol = 0;
rc = SQLITE_OK;
}else{
pCsr->isEof = 1;
}
return rc;
}
/*
** xFilter - Initialize a cursor to point at the start of its data.
*/
static int fts3auxFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
int rc;
int isScan;
UNUSED_PARAMETER(nVal);
assert( idxStr==0 );
assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0
|| idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT
|| idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT)
);
isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT);
/* In case this cursor is being reused, close and zero it. */
testcase(pCsr->filter.zTerm);
sqlite3Fts3SegReaderFinish(&pCsr->csr);
sqlite3_free((void *)pCsr->filter.zTerm);
sqlite3_free(pCsr->aStat);
memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){
const unsigned char *zStr = sqlite3_value_text(apVal[0]);
if( zStr ){
pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
}
}
if( idxNum&FTS4AUX_LE_CONSTRAINT ){
int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
}
rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
);
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
}
if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
return rc;
}
/*
** xEof - Return true if the cursor is at EOF, or false otherwise.
*/
static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
return pCsr->isEof;
}
/*
** xColumn - Return a column value.
*/
static int fts3auxColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
assert( p->isEof==0 );
if( iCol==0 ){ /* Column "term" */
sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
}else if( iCol==1 ){ /* Column "col" */
if( p->iCol ){
sqlite3_result_int(pContext, p->iCol-1);
}else{
sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC);
}
}else if( iCol==2 ){ /* Column "documents" */
sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc);
}else{ /* Column "occurrences" */
sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc);
}
return SQLITE_OK;
}
/*
** xRowid - Return the current rowid for the cursor.
*/
static int fts3auxRowidMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite_int64 *pRowid /* OUT: Rowid value */
){
Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
*pRowid = pCsr->iRowid;
return SQLITE_OK;
}
/*
** Register the fts3aux module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts3InitAux(sqlite3 *db){
static const sqlite3_module fts3aux_module = {
0, /* iVersion */
fts3auxConnectMethod, /* xCreate */
fts3auxConnectMethod, /* xConnect */
fts3auxBestIndexMethod, /* xBestIndex */
fts3auxDisconnectMethod, /* xDisconnect */
fts3auxDisconnectMethod, /* xDestroy */
fts3auxOpenMethod, /* xOpen */
fts3auxCloseMethod, /* xClose */
fts3auxFilterMethod, /* xFilter */
fts3auxNextMethod, /* xNext */
fts3auxEofMethod, /* xEof */
fts3auxColumnMethod, /* xColumn */
fts3auxRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
0 /* xRename */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-937
View File
@@ -1,937 +0,0 @@
/*
** 2008 Nov 28
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This module contains code that implements a parser for fts3 query strings
** (the right-hand argument to the MATCH operator). Because the supported
** syntax is relatively simple, the whole tokenizer/parser system is
** hand-coded.
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
/*
** By default, this module parses the legacy syntax that has been
** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
** is defined, then it uses the new syntax. The differences between
** the new and the old syntaxes are:
**
** a) The new syntax supports parenthesis. The old does not.
**
** b) The new syntax supports the AND and NOT operators. The old does not.
**
** c) The old syntax supports the "-" token qualifier. This is not
** supported by the new syntax (it is replaced by the NOT operator).
**
** d) When using the old syntax, the OR operator has a greater precedence
** than an implicit AND. When using the new, both implicity and explicit
** AND operators have a higher precedence than OR.
**
** If compiled with SQLITE_TEST defined, then this module exports the
** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
** to zero causes the module to use the old syntax. If it is set to
** non-zero the new syntax is activated. This is so both syntaxes can
** be tested using a single build of testfixture.
**
** The following describes the syntax supported by the fts3 MATCH
** operator in a similar format to that used by the lemon parser
** generator. This module does not use actually lemon, it uses a
** custom parser.
**
** query ::= andexpr (OR andexpr)*.
**
** andexpr ::= notexpr (AND? notexpr)*.
**
** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
** notexpr ::= LP query RP.
**
** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
**
** distance_opt ::= .
** distance_opt ::= / INTEGER.
**
** phrase ::= TOKEN.
** phrase ::= COLUMN:TOKEN.
** phrase ::= "TOKEN TOKEN TOKEN...".
*/
#ifdef SQLITE_TEST
int sqlite3_fts3_enable_parentheses = 0;
#else
# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
# define sqlite3_fts3_enable_parentheses 1
# else
# define sqlite3_fts3_enable_parentheses 0
# endif
#endif
/*
** Default span for NEAR operators.
*/
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
#include "fts3Int.h"
#include <string.h>
#include <assert.h>
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
const char **azCol; /* Array of column names for fts3 table */
int nCol; /* Number of entries in azCol[] */
int iDefaultCol; /* Default column to query */
sqlite3_context *pCtx; /* Write error message here */
int nNest; /* Number of nested brackets */
};
/*
** This function is equivalent to the standard isspace() function.
**
** The standard isspace() can be awkward to use safely, because although it
** is defined to accept an argument of type int, its behaviour when passed
** an integer that falls outside of the range of the unsigned char type
** is undefined (and sometimes, "undefined" means segfault). This wrapper
** is defined to accept an argument of type char, and always returns 0 for
** any values that fall outside of the range of the unsigned char type (i.e.
** negative values).
*/
static int fts3isspace(char c){
return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
}
/*
** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
** zero the memory before returning a pointer to it. If unsuccessful,
** return NULL.
*/
static void *fts3MallocZero(int nByte){
void *pRet = sqlite3_malloc(nByte);
if( pRet ) memset(pRet, 0, nByte);
return pRet;
}
/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is
** reached before a token is found, set *ppExpr to zero. It is the
** responsibility of the caller to eventually deallocate the allocated
** Fts3Expr structure (if any) by passing it to sqlite3_free().
**
** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
** fails.
*/
static int getNextToken(
ParseContext *pParse, /* fts3 query parse context */
int iCol, /* Value for Fts3Phrase.iColumn */
const char *z, int n, /* Input string */
Fts3Expr **ppExpr, /* OUT: expression */
int *pnConsumed /* OUT: Number of bytes consumed */
){
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
int rc;
sqlite3_tokenizer_cursor *pCursor;
Fts3Expr *pRet = 0;
int nConsumed = 0;
rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken, iStart, iEnd, iPosition;
int nByte; /* total space to allocate */
pCursor->pTokenizer = pTokenizer;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
pRet = (Fts3Expr *)fts3MallocZero(nByte);
if( !pRet ){
rc = SQLITE_NOMEM;
}else{
pRet->eType = FTSQUERY_PHRASE;
pRet->pPhrase = (Fts3Phrase *)&pRet[1];
pRet->pPhrase->nToken = 1;
pRet->pPhrase->iColumn = iCol;
pRet->pPhrase->aToken[0].n = nToken;
pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
if( iEnd<n && z[iEnd]=='*' ){
pRet->pPhrase->aToken[0].isPrefix = 1;
iEnd++;
}
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
pRet->pPhrase->isNot = 1;
}
}
nConsumed = iEnd;
}
pModule->xClose(pCursor);
}
*pnConsumed = nConsumed;
*ppExpr = pRet;
return rc;
}
/*
** Enlarge a memory allocation. If an out-of-memory allocation occurs,
** then free the old allocation.
*/
static void *fts3ReallocOrFree(void *pOrig, int nNew){
void *pRet = sqlite3_realloc(pOrig, nNew);
if( !pRet ){
sqlite3_free(pOrig);
}
return pRet;
}
/*
** Buffer zInput, length nInput, contains the contents of a quoted string
** that appeared as part of an fts3 query expression. Neither quote character
** is included in the buffer. This function attempts to tokenize the entire
** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
** containing the results.
**
** If successful, SQLITE_OK is returned and *ppExpr set to point at the
** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
** to 0.
*/
static int getNextString(
ParseContext *pParse, /* fts3 query parse context */
const char *zInput, int nInput, /* Input string */
Fts3Expr **ppExpr /* OUT: expression */
){
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
int rc;
Fts3Expr *p = 0;
sqlite3_tokenizer_cursor *pCursor = 0;
char *zTemp = 0;
int nTemp = 0;
rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
if( rc==SQLITE_OK ){
int ii;
pCursor->pTokenizer = pTokenizer;
for(ii=0; rc==SQLITE_OK; ii++){
const char *zToken;
int nToken, iBegin, iEnd, iPos;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
if( rc==SQLITE_OK ){
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
if( !p || !zTemp ){
goto no_mem;
}
if( ii==0 ){
memset(p, 0, nByte);
p->pPhrase = (Fts3Phrase *)&p[1];
}
p->pPhrase = (Fts3Phrase *)&p[1];
memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
p->pPhrase->nToken = ii+1;
p->pPhrase->aToken[ii].n = nToken;
memcpy(&zTemp[nTemp], zToken, nToken);
nTemp += nToken;
if( iEnd<nInput && zInput[iEnd]=='*' ){
p->pPhrase->aToken[ii].isPrefix = 1;
}else{
p->pPhrase->aToken[ii].isPrefix = 0;
}
}
}
pModule->xClose(pCursor);
pCursor = 0;
}
if( rc==SQLITE_DONE ){
int jj;
char *zNew = NULL;
int nNew = 0;
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
p = fts3ReallocOrFree(p, nByte + nTemp);
if( !p ){
goto no_mem;
}
if( zTemp ){
zNew = &(((char *)p)[nByte]);
memcpy(zNew, zTemp, nTemp);
}else{
memset(p, 0, nByte+nTemp);
}
p->pPhrase = (Fts3Phrase *)&p[1];
for(jj=0; jj<p->pPhrase->nToken; jj++){
p->pPhrase->aToken[jj].z = &zNew[nNew];
nNew += p->pPhrase->aToken[jj].n;
}
sqlite3_free(zTemp);
p->eType = FTSQUERY_PHRASE;
p->pPhrase->iColumn = pParse->iDefaultCol;
rc = SQLITE_OK;
}
*ppExpr = p;
return rc;
no_mem:
if( pCursor ){
pModule->xClose(pCursor);
}
sqlite3_free(zTemp);
sqlite3_free(p);
*ppExpr = 0;
return SQLITE_NOMEM;
}
/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
/*
** The output variable *ppExpr is populated with an allocated Fts3Expr
** structure, or set to 0 if the end of the input buffer is reached.
**
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
** If SQLITE_ERROR is returned, pContext is populated with an error message.
*/
static int getNextNode(
ParseContext *pParse, /* fts3 query parse context */
const char *z, int n, /* Input string */
Fts3Expr **ppExpr, /* OUT: expression */
int *pnConsumed /* OUT: Number of bytes consumed */
){
static const struct Fts3Keyword {
char *z; /* Keyword text */
unsigned char n; /* Length of the keyword */
unsigned char parenOnly; /* Only valid in paren mode */
unsigned char eType; /* Keyword code */
} aKeyword[] = {
{ "OR" , 2, 0, FTSQUERY_OR },
{ "AND", 3, 1, FTSQUERY_AND },
{ "NOT", 3, 1, FTSQUERY_NOT },
{ "NEAR", 4, 0, FTSQUERY_NEAR }
};
int ii;
int iCol;
int iColLen;
int rc;
Fts3Expr *pRet = 0;
const char *zInput = z;
int nInput = n;
/* Skip over any whitespace before checking for a keyword, an open or
** close bracket, or a quoted string.
*/
while( nInput>0 && fts3isspace(*zInput) ){
nInput--;
zInput++;
}
if( nInput==0 ){
return SQLITE_DONE;
}
/* See if we are dealing with a keyword. */
for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
const struct Fts3Keyword *pKey = &aKeyword[ii];
if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
continue;
}
if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
int nKey = pKey->n;
char cNext;
/* If this is a "NEAR" keyword, check for an explicit nearness. */
if( pKey->eType==FTSQUERY_NEAR ){
assert( nKey==4 );
if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
nNear = 0;
for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
nNear = nNear * 10 + (zInput[nKey] - '0');
}
}
}
/* At this point this is probably a keyword. But for that to be true,
** the next byte must contain either whitespace, an open or close
** parenthesis, a quote character, or EOF.
*/
cNext = zInput[nKey];
if( fts3isspace(cNext)
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
if( !pRet ){
return SQLITE_NOMEM;
}
pRet->eType = pKey->eType;
pRet->nNear = nNear;
*ppExpr = pRet;
*pnConsumed = (int)((zInput - z) + nKey);
return SQLITE_OK;
}
/* Turns out that wasn't a keyword after all. This happens if the
** user has supplied a token such as "ORacle". Continue.
*/
}
}
/* Check for an open bracket. */
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed;
pParse->nNest++;
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){
rc = SQLITE_DONE;
}
*pnConsumed = (int)((zInput - z) + 1 + nConsumed);
return rc;
}
/* Check for a close bracket. */
if( *zInput==')' ){
pParse->nNest--;
*pnConsumed = (int)((zInput - z) + 1);
return SQLITE_DONE;
}
}
/* See if we are dealing with a quoted phrase. If this is the case, then
** search for the closing quote and pass the whole string to getNextString()
** for processing. This is easy to do, as fts3 has no syntax for escaping
** a quote character embedded in a string.
*/
if( *zInput=='"' ){
for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
*pnConsumed = (int)((zInput - z) + ii + 1);
if( ii==nInput ){
return SQLITE_ERROR;
}
return getNextString(pParse, &zInput[1], ii-1, ppExpr);
}
/* If control flows to this point, this must be a regular token, or
** the end of the input. Read a regular token using the sqlite3_tokenizer
** interface. Before doing so, figure out if there is an explicit
** column specifier for the token.
**
** TODO: Strangely, it is not possible to associate a column specifier
** with a quoted phrase, only with a single token. Not sure if this was
** an implementation artifact or an intentional decision when fts3 was
** first implemented. Whichever it was, this module duplicates the
** limitation.
*/
iCol = pParse->iDefaultCol;
iColLen = 0;
for(ii=0; ii<pParse->nCol; ii++){
const char *zStr = pParse->azCol[ii];
int nStr = (int)strlen(zStr);
if( nInput>nStr && zInput[nStr]==':'
&& sqlite3_strnicmp(zStr, zInput, nStr)==0
){
iCol = ii;
iColLen = (int)((zInput - z) + nStr + 1);
break;
}
}
rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
*pnConsumed += iColLen;
return rc;
}
/*
** The argument is an Fts3Expr structure for a binary operator (any type
** except an FTSQUERY_PHRASE). Return an integer value representing the
** precedence of the operator. Lower values have a higher precedence (i.e.
** group more tightly). For example, in the C language, the == operator
** groups more tightly than ||, and would therefore have a higher precedence.
**
** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
** is defined), the order of the operators in precedence from highest to
** lowest is:
**
** NEAR
** NOT
** AND (including implicit ANDs)
** OR
**
** Note that when using the old query syntax, the OR operator has a higher
** precedence than the AND operator.
*/
static int opPrecedence(Fts3Expr *p){
assert( p->eType!=FTSQUERY_PHRASE );
if( sqlite3_fts3_enable_parentheses ){
return p->eType;
}else if( p->eType==FTSQUERY_NEAR ){
return 1;
}else if( p->eType==FTSQUERY_OR ){
return 2;
}
assert( p->eType==FTSQUERY_AND );
return 3;
}
/*
** Argument ppHead contains a pointer to the current head of a query
** expression tree being parsed. pPrev is the expression node most recently
** inserted into the tree. This function adds pNew, which is always a binary
** operator node, into the expression tree based on the relative precedence
** of pNew and the existing nodes of the tree. This may result in the head
** of the tree changing, in which case *ppHead is set to the new root node.
*/
static void insertBinaryOperator(
Fts3Expr **ppHead, /* Pointer to the root node of a tree */
Fts3Expr *pPrev, /* Node most recently inserted into the tree */
Fts3Expr *pNew /* New binary node to insert into expression tree */
){
Fts3Expr *pSplit = pPrev;
while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
pSplit = pSplit->pParent;
}
if( pSplit->pParent ){
assert( pSplit->pParent->pRight==pSplit );
pSplit->pParent->pRight = pNew;
pNew->pParent = pSplit->pParent;
}else{
*ppHead = pNew;
}
pNew->pLeft = pSplit;
pSplit->pParent = pNew;
}
/*
** Parse the fts3 query expression found in buffer z, length n. This function
** returns either when the end of the buffer is reached or an unmatched
** closing bracket - ')' - is encountered.
**
** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
** parsed form of the expression and *pnConsumed is set to the number of
** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
** (out of memory error) or SQLITE_ERROR (parse error) is returned.
*/
static int fts3ExprParse(
ParseContext *pParse, /* fts3 query parse context */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr, /* OUT: Parsed query structure */
int *pnConsumed /* OUT: Number of bytes consumed */
){
Fts3Expr *pRet = 0;
Fts3Expr *pPrev = 0;
Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
int nIn = n;
const char *zIn = z;
int rc = SQLITE_OK;
int isRequirePhrase = 1;
while( rc==SQLITE_OK ){
Fts3Expr *p = 0;
int nByte = 0;
rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
if( rc==SQLITE_OK ){
int isPhrase;
if( !sqlite3_fts3_enable_parentheses
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
){
/* Create an implicit NOT operator. */
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
if( pNotBranch ){
pNot->pLeft = pNotBranch;
}
pNotBranch = pNot;
p = pPrev;
}else{
int eType = p->eType;
assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
/* The isRequirePhrase variable is set to true if a phrase or
** an expression contained in parenthesis is required. If a
** binary operator (AND, OR, NOT or NEAR) is encounted when
** isRequirePhrase is set, this is a syntax error.
*/
if( !isPhrase && isRequirePhrase ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase && !isRequirePhrase ){
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
}
/* This test catches attempts to make either operand of a NEAR
** operator something other than a phrase. For example, either of
** the following:
**
** (bracketed expression) NEAR phrase
** phrase NEAR (bracketed expression)
**
** Return an error in either case.
*/
if( pPrev && (
(eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
|| (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
)){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase ){
if( pRet ){
assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
pPrev->pRight = p;
p->pParent = pPrev;
}else{
pRet = p;
}
}else{
insertBinaryOperator(&pRet, pPrev, p);
}
isRequirePhrase = !isPhrase;
}
assert( nByte>0 );
}
assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
nIn -= nByte;
zIn += nByte;
pPrev = p;
}
if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
rc = SQLITE_ERROR;
}
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
if( !pRet ){
rc = SQLITE_ERROR;
}else{
Fts3Expr *pIter = pNotBranch;
while( pIter->pLeft ){
pIter = pIter->pLeft;
}
pIter->pLeft = pRet;
pRet = pNotBranch;
}
}
}
*pnConsumed = n - nIn;
exprparse_out:
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRet);
sqlite3Fts3ExprFree(pNotBranch);
pRet = 0;
}
*ppExpr = pRet;
return rc;
}
/*
** Parameters z and n contain a pointer to and length of a buffer containing
** an fts3 query expression, respectively. This function attempts to parse the
** query expression and create a tree of Fts3Expr structures representing the
** parsed expression. If successful, *ppExpr is set to point to the head
** of the parsed expression tree and SQLITE_OK is returned. If an error
** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
** error) is returned and *ppExpr is set to 0.
**
** If parameter n is a negative number, then z is assumed to point to a
** nul-terminated string and the length is determined using strlen().
**
** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
** use to normalize query tokens while parsing the expression. The azCol[]
** array, which is assumed to contain nCol entries, should contain the names
** of each column in the target fts3 table, in order from left to right.
** Column names must be nul-terminated strings.
**
** The iDefaultCol parameter should be passed the index of the table column
** that appears on the left-hand-side of the MATCH operator (the default
** column to match against for tokens for which a column name is not explicitly
** specified as part of the query string), or -1 if tokens may by default
** match any table column.
*/
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
char **azCol, /* Array of column names for fts3 table */
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr /* OUT: Parsed query structure */
){
int nParsed;
int rc;
ParseContext sParse;
sParse.pTokenizer = pTokenizer;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.nNest = 0;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
/* Check for mismatched parenthesis */
if( rc==SQLITE_OK && sParse.nNest ){
rc = SQLITE_ERROR;
sqlite3Fts3ExprFree(*ppExpr);
*ppExpr = 0;
}
return rc;
}
/*
** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
*/
void sqlite3Fts3ExprFree(Fts3Expr *p){
if( p ){
sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight);
sqlite3_free(p->aDoclist);
sqlite3_free(p);
}
}
/****************************************************************************
*****************************************************************************
** Everything after this point is just test code.
*/
#ifdef SQLITE_TEST
#include <stdio.h>
/*
** Function to query the hash-table of tokenizers (see README.tokenizers).
*/
static int queryTestTokenizer(
sqlite3 *db,
const char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
/*
** Return a pointer to a buffer containing a text representation of the
** expression passed as the first argument. The buffer is obtained from
** sqlite3_malloc(). It is the responsibility of the caller to use
** sqlite3_free() to release the memory. If an OOM condition is encountered,
** NULL is returned.
**
** If the second argument is not NULL, then its contents are prepended to
** the returned expression text and then freed using sqlite3_free().
*/
static char *exprToString(Fts3Expr *pExpr, char *zBuf){
switch( pExpr->eType ){
case FTSQUERY_PHRASE: {
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
zBuf = sqlite3_mprintf(
"%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
for(i=0; zBuf && i<pPhrase->nToken; i++){
zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
pPhrase->aToken[i].n, pPhrase->aToken[i].z,
(pPhrase->aToken[i].isPrefix?"+":"")
);
}
return zBuf;
}
case FTSQUERY_NEAR:
zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
break;
case FTSQUERY_NOT:
zBuf = sqlite3_mprintf("%zNOT ", zBuf);
break;
case FTSQUERY_AND:
zBuf = sqlite3_mprintf("%zAND ", zBuf);
break;
case FTSQUERY_OR:
zBuf = sqlite3_mprintf("%zOR ", zBuf);
break;
}
if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
return zBuf;
}
/*
** This is the implementation of a scalar SQL function used to test the
** expression parser. It should be called as follows:
**
** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
**
** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
** to parse the query expression (see README.tokenizers). The second argument
** is the query expression to parse. Each subsequent argument is the name
** of a column of the fts3 table that the query expression may refer to.
** For example:
**
** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
*/
static void fts3ExprTest(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
sqlite3_tokenizer_module const *pModule = 0;
sqlite3_tokenizer *pTokenizer = 0;
int rc;
char **azCol = 0;
const char *zExpr;
int nExpr;
int nCol;
int ii;
Fts3Expr *pExpr;
char *zBuf = 0;
sqlite3 *db = sqlite3_context_db_handle(context);
if( argc<3 ){
sqlite3_result_error(context,
"Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
);
return;
}
rc = queryTestTokenizer(db,
(const char *)sqlite3_value_text(argv[0]), &pModule);
if( rc==SQLITE_NOMEM ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}else if( !pModule ){
sqlite3_result_error(context, "No such tokenizer module", -1);
goto exprtest_out;
}
rc = pModule->xCreate(0, 0, &pTokenizer);
assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
if( rc==SQLITE_NOMEM ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}
pTokenizer->pModule = pModule;
zExpr = (const char *)sqlite3_value_text(argv[1]);
nExpr = sqlite3_value_bytes(argv[1]);
nCol = argc-2;
azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
if( !azCol ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}
for(ii=0; ii<nCol; ii++){
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
}else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
sqlite3_result_error_nomem(context);
}else{
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
sqlite3_free(zBuf);
}
sqlite3Fts3ExprFree(pExpr);
exprtest_out:
if( pModule && pTokenizer ){
rc = pModule->xDestroy(pTokenizer);
}
sqlite3_free(azCol);
}
/*
** Register the query expression parser test function fts3_exprtest()
** with database connection db.
*/
int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
return sqlite3_create_function(
db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
);
}
#endif
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-383
View File
@@ -1,383 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS3 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS3 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "sqlite3.h"
#include "fts3_hash.h"
/*
** Malloc and Free functions
*/
static void *fts3HashMalloc(int n){
void *p = sqlite3_malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
static void fts3HashFree(void *p){
sqlite3_free(p);
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants
** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void sqlite3Fts3HashClear(Fts3Hash *pH){
Fts3HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
fts3HashFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
Fts3HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
fts3HashFree(elem->pKey);
}
fts3HashFree(elem);
elem = next_elem;
}
pH->count = 0;
}
/*
** Hash and comparison functions when the mode is FTS3_HASH_STRING
*/
static int fts3StrHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is FTS3_HASH_BINARY
*/
static int fts3BinHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "ftsHashFunction". The function takes a
** single parameter "keyClass". The return value of ftsHashFunction()
** is a pointer to another function. Specifically, the return value
** of ftsHashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*ftsHashFunction(int keyClass))(const void*,int){
if( keyClass==FTS3_HASH_STRING ){
return &fts3StrHash;
}else{
assert( keyClass==FTS3_HASH_BINARY );
return &fts3BinHash;
}
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
if( keyClass==FTS3_HASH_STRING ){
return &fts3StrCompare;
}else{
assert( keyClass==FTS3_HASH_BINARY );
return &fts3BinCompare;
}
}
/* Link an element into the hash table
*/
static void fts3HashInsertElement(
Fts3Hash *pH, /* The complete hash table */
struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
Fts3HashElem *pNew /* The element to be inserted */
){
Fts3HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
**
** Return non-zero if a memory allocation error occurs.
*/
static int fts3Rehash(Fts3Hash *pH, int new_size){
struct _fts3ht *new_ht; /* The new hash table */
Fts3HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
if( new_ht==0 ) return 1;
fts3HashFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = ftsHashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
fts3HashInsertElement(pH, &new_ht[h], elem);
}
return 0;
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static Fts3HashElem *fts3FindElementByHash(
const Fts3Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
Fts3HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts3ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = ftsCompareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void fts3RemoveElementByHash(
Fts3Hash *pH, /* The pH containing "elem" */
Fts3HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts3ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
fts3HashFree(elem->pKey);
}
fts3HashFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
fts3HashClear(pH);
}
}
Fts3HashElem *sqlite3Fts3HashFindElem(
const Fts3Hash *pH,
const void *pKey,
int nKey
){
int h; /* A hash on key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = ftsHashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
}
/*
** Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
Fts3HashElem *pElem; /* The element that matches key (if any) */
pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey);
return pElem ? pElem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts3HashInsert(
Fts3Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
Fts3HashElem *elem; /* Used to loop thru the element list */
Fts3HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = ftsHashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = fts3FindElementByHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
fts3RemoveElementByHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
if( (pH->htsize==0 && fts3Rehash(pH,8))
|| (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2))
){
pH->count = 0;
return data;
}
assert( pH->htsize>0 );
new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = fts3HashMalloc( nKey );
if( new_elem->pKey==0 ){
fts3HashFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
fts3HashInsertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-112
View File
@@ -1,112 +0,0 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _FTS3_HASH_H_
#define _FTS3_HASH_H_
/* Forward declarations of structures. */
typedef struct Fts3Hash Fts3Hash;
typedef struct Fts3HashElem Fts3HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct Fts3Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
Fts3HashElem *first; /* The first element of the array */
int htsize; /* Number of buckets in the hash table */
struct _fts3ht { /* the hash table */
int count; /* Number of entries with this hash */
Fts3HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct Fts3HashElem {
Fts3HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
*/
#define FTS3_HASH_STRING 1
#define FTS3_HASH_BINARY 2
/*
** Access routines. To delete, insert a NULL pointer.
*/
void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey);
void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
void sqlite3Fts3HashClear(Fts3Hash*);
Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int);
/*
** Shorthand for the functions above
*/
#define fts3HashInit sqlite3Fts3HashInit
#define fts3HashInsert sqlite3Fts3HashInsert
#define fts3HashFind sqlite3Fts3HashFind
#define fts3HashClear sqlite3Fts3HashClear
#define fts3HashFindElem sqlite3Fts3HashFindElem
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** Fts3Hash h;
** Fts3HashElem *p;
** ...
** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
** SomeStructure *pData = fts3HashData(p);
** // do something with pData
** }
*/
#define fts3HashFirst(H) ((H)->first)
#define fts3HashNext(E) ((E)->next)
#define fts3HashData(E) ((E)->data)
#define fts3HashKey(E) ((E)->pKey)
#define fts3HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define fts3HashCount(H) ((H)->count)
#endif /* _FTS3_HASH_H_ */
-260
View File
@@ -1,260 +0,0 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements a tokenizer for fts3 based on the ICU library.
**
** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_ENABLE_ICU
#include <assert.h>
#include <string.h>
#include "fts3_tokenizer.h"
#include <unicode/ubrk.h>
#include <unicode/ucol.h>
#include <unicode/ustring.h>
#include <unicode/utf16.h>
typedef struct IcuTokenizer IcuTokenizer;
typedef struct IcuCursor IcuCursor;
struct IcuTokenizer {
sqlite3_tokenizer base;
char *zLocale;
};
struct IcuCursor {
sqlite3_tokenizer_cursor base;
UBreakIterator *pIter; /* ICU break-iterator object */
int nChar; /* Number of UChar elements in pInput */
UChar *aChar; /* Copy of input using utf-16 encoding */
int *aOffset; /* Offsets of each character in utf-8 input */
int nBuffer;
char *zBuffer;
int iToken;
};
/*
** Create a new tokenizer instance.
*/
static int icuCreate(
int argc, /* Number of entries in argv[] */
const char * const *argv, /* Tokenizer creation arguments */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
){
IcuTokenizer *p;
int n = 0;
if( argc>0 ){
n = strlen(argv[0])+1;
}
p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
if( !p ){
return SQLITE_NOMEM;
}
memset(p, 0, sizeof(IcuTokenizer));
if( n ){
p->zLocale = (char *)&p[1];
memcpy(p->zLocale, argv[0], n);
}
*ppTokenizer = (sqlite3_tokenizer *)p;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int icuDestroy(sqlite3_tokenizer *pTokenizer){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
sqlite3_free(p);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int icuOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, /* Input string */
int nInput, /* Length of zInput in bytes */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
IcuCursor *pCsr;
const int32_t opt = U_FOLD_CASE_DEFAULT;
UErrorCode status = U_ZERO_ERROR;
int nChar;
UChar32 c;
int iInput = 0;
int iOut = 0;
*ppCursor = 0;
if( nInput<0 ){
nInput = strlen(zInput);
}
nChar = nInput+1;
pCsr = (IcuCursor *)sqlite3_malloc(
sizeof(IcuCursor) + /* IcuCursor */
nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
(nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
);
if( !pCsr ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(IcuCursor));
pCsr->aChar = (UChar *)&pCsr[1];
pCsr->aOffset = (int *)&pCsr->aChar[nChar];
pCsr->aOffset[iOut] = iInput;
U8_NEXT(zInput, iInput, nInput, c);
while( c>0 ){
int isError = 0;
c = u_foldCase(c, opt);
U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
if( isError ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->aOffset[iOut] = iInput;
if( iInput<nInput ){
U8_NEXT(zInput, iInput, nInput, c);
}else{
c = 0;
}
}
pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
if( !U_SUCCESS(status) ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->nChar = iOut;
ubrk_first(pCsr->pIter);
*ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to icuOpen().
*/
static int icuClose(sqlite3_tokenizer_cursor *pCursor){
IcuCursor *pCsr = (IcuCursor *)pCursor;
ubrk_close(pCsr->pIter);
sqlite3_free(pCsr->zBuffer);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor.
*/
static int icuNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
IcuCursor *pCsr = (IcuCursor *)pCursor;
int iStart = 0;
int iEnd = 0;
int nByte = 0;
while( iStart==iEnd ){
UChar32 c;
iStart = ubrk_current(pCsr->pIter);
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
}
do {
UErrorCode status = U_ZERO_ERROR;
if( nByte ){
char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
if( !zNew ){
return SQLITE_NOMEM;
}
pCsr->zBuffer = zNew;
pCsr->nBuffer = nByte;
}
u_strToUTF8(
pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
&pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
&status /* Output success/failure */
);
} while( nByte>pCsr->nBuffer );
*ppToken = pCsr->zBuffer;
*pnBytes = nByte;
*piStartOffset = pCsr->aOffset[iStart];
*piEndOffset = pCsr->aOffset[iEnd];
*piPosition = pCsr->iToken++;
return SQLITE_OK;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
};
/*
** Set *ppModule to point at the implementation of the ICU tokenizer.
*/
void sqlite3Fts3IcuTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &icuTokenizerModule;
}
#endif /* defined(SQLITE_ENABLE_ICU) */
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-646
View File
@@ -1,646 +0,0 @@
/*
** 2006 September 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS3 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS3 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include "fts3Int.h"
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fts3_tokenizer.h"
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
sqlite3_tokenizer base; /* Base class */
} porter_tokenizer;
/*
** Class derived from sqlit3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *zInput; /* input we are tokenizing */
int nInput; /* size of the input */
int iOffset; /* current position in zInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nAllocated; /* space allocated to zToken buffer */
} porter_tokenizer_cursor;
/*
** Create a new tokenizer instance.
*/
static int porterCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
porter_tokenizer *t;
UNUSED_PARAMETER(argc);
UNUSED_PARAMETER(argv);
t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int porterDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is zInput[0..nInput-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int porterOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, int nInput, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
porter_tokenizer_cursor *c;
UNUSED_PARAMETER(pTokenizer);
c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->zInput = zInput;
if( zInput==0 ){
c->nInput = 0;
}else if( nInput<0 ){
c->nInput = (int)strlen(zInput);
}else{
c->nInput = nInput;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** porterOpen() above.
*/
static int porterClose(sqlite3_tokenizer_cursor *pCursor){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
sqlite3_free(c->zToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Vowel or consonant
*/
static const char cType[] = {
0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
1, 1, 1, 2, 1
};
/*
** isConsonant() and isVowel() determine if their first character in
** the string they point to is a consonant or a vowel, according
** to Porter ruls.
**
** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
** 'Y' is a consonant unless it follows another consonant,
** in which case it is a vowel.
**
** In these routine, the letters are in reverse order. So the 'y' rule
** is that 'y' is a consonant unless it is followed by another
** consonent.
*/
static int isVowel(const char*);
static int isConsonant(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return j;
return z[1]==0 || isVowel(z + 1);
}
static int isVowel(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return 1-j;
return isConsonant(z + 1);
}
/*
** Let any sequence of one or more vowels be represented by V and let
** C be sequence of one or more consonants. Then every word can be
** represented as:
**
** [C] (VC){m} [V]
**
** In prose: A word is an optional consonant followed by zero or
** vowel-consonant pairs followed by an optional vowel. "m" is the
** number of vowel consonant pairs. This routine computes the value
** of m for the first i bytes of a word.
**
** Return true if the m-value for z is 1 or more. In other words,
** return true if z contains at least one vowel that is followed
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/* Like mgt0 above except we are looking for a value of m which is
** exactly 1
*/
static int m_eq_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 1;
while( isConsonant(z) ){ z++; }
return *z==0;
}
/* Like mgt0 above except we are looking for a value of m>1 instead
** or m>0
*/
static int m_gt_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if there is a vowel anywhere within z[0..n-1]
*/
static int hasVowel(const char *z){
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if the word ends in a double consonant.
**
** The text is reversed here. So we are really looking at
** the first two characters of z[].
*/
static int doubleConsonant(const char *z){
return isConsonant(z) && z[0]==z[1];
}
/*
** Return TRUE if the word ends with three letters which
** are consonant-vowel-consonent and where the final consonant
** is not 'w', 'x', or 'y'.
**
** The word is reversed here. So we are really checking the
** first three letters and the first one cannot be in [wxy].
*/
static int star_oh(const char *z){
return
isConsonant(z) &&
z[0]!='w' && z[0]!='x' && z[0]!='y' &&
isVowel(z+1) &&
isConsonant(z+2);
}
/*
** If the word ends with zFrom and xCond() is true for the stem
** of the word that preceeds the zFrom ending, then change the
** ending to zTo.
**
** The input word *pz and zFrom are both in reverse order. zTo
** is in normal order.
**
** Return TRUE if zFrom matches. Return FALSE if zFrom does not
** match. Not that TRUE is returned even if xCond() fails and
** no substitution occurs.
*/
static int stem(
char **pz, /* The word being stemmed (Reversed) */
const char *zFrom, /* If the ending matches this... (Reversed) */
const char *zTo, /* ... change the ending to this (not reversed) */
int (*xCond)(const char*) /* Condition that must be true */
){
char *z = *pz;
while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
if( *zFrom!=0 ) return 0;
if( xCond && !xCond(z) ) return 1;
while( *zTo ){
*(--z) = *(zTo++);
}
*pz = z;
return 1;
}
/*
** This is the fallback stemmer used when the porter stemmer is
** inappropriate. The input word is copied into the output with
** US-ASCII case folding. If the input word is too long (more
** than 20 bytes if it contains no digits or more than 6 bytes if
** it contains digits) then word is truncated to 20 or 6 bytes
** by taking 10 or 3 bytes from the beginning and end.
*/
static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, mx, j;
int hasDigit = 0;
for(i=0; i<nIn; i++){
char c = zIn[i];
if( c>='A' && c<='Z' ){
zOut[i] = c - 'A' + 'a';
}else{
if( c>='0' && c<='9' ) hasDigit = 1;
zOut[i] = c;
}
}
mx = hasDigit ? 3 : 10;
if( nIn>mx*2 ){
for(j=mx, i=nIn-mx; i<nIn; i++, j++){
zOut[j] = zOut[i];
}
i = j;
}
zOut[i] = 0;
*pnOut = i;
}
/*
** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
** zOut is at least big enough to hold nIn bytes. Write the actual
** size of the output word (exclusive of the '\0' terminator) into *pnOut.
**
** Any upper-case characters in the US-ASCII character set ([A-Z])
** are converted to lower case. Upper-case UTF characters are
** unchanged.
**
** Words that are longer than about 20 bytes are stemmed by retaining
** a few bytes from the beginning and the end of the word. If the
** word contains digits, 3 bytes are taken from the beginning and
** 3 bytes from the end. For long words without digits, 10 bytes
** are taken from each end. US-ASCII case folding still applies.
**
** If the input word contains not digits but does characters not
** in [a-zA-Z] then no stemming is attempted and this routine just
** copies the input into the input into the output with US-ASCII
** case folding.
**
** Stemming never increases the length of the word. So there is
** no chance of overflowing the zOut buffer.
*/
static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, j;
char zReverse[28];
char *z, *z2;
if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
char c = zIn[i];
if( c>='A' && c<='Z' ){
zReverse[j] = c + 'a' - 'A';
}else if( c>='a' && c<='z' ){
zReverse[j] = c;
}else{
/* The use of a character not in [a-zA-Z] means that we fallback
** to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
}
memset(&zReverse[sizeof(zReverse)-5], 0, 5);
z = &zReverse[j+1];
/* Step 1a */
if( z[0]=='s' ){
if(
!stem(&z, "sess", "ss", 0) &&
!stem(&z, "sei", "i", 0) &&
!stem(&z, "ss", "ss", 0)
){
z++;
}
}
/* Step 1b */
z2 = z;
if( stem(&z, "dee", "ee", m_gt_0) ){
/* Do nothing. The work was all in the test */
}else if(
(stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
&& z!=z2
){
if( stem(&z, "ta", "ate", 0) ||
stem(&z, "lb", "ble", 0) ||
stem(&z, "zi", "ize", 0) ){
/* Do nothing. The work was all in the test */
}else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
z++;
}else if( m_eq_1(z) && star_oh(z) ){
*(--z) = 'e';
}
}
/* Step 1c */
if( z[0]=='y' && hasVowel(z+1) ){
z[0] = 'i';
}
/* Step 2 */
switch( z[1] ){
case 'a':
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
break;
case 'g':
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
break;
}
/* Step 4 */
switch( z[1] ){
case 'a':
if( z[0]=='l' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'c':
if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'e':
if( z[0]=='r' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'i':
if( z[0]=='c' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'l':
if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'n':
if( z[0]=='t' ){
if( z[2]=='a' ){
if( m_gt_1(z+3) ){
z += 3;
}
}else if( z[2]=='e' ){
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
case 'o':
if( z[0]=='u' ){
if( m_gt_1(z+2) ){
z += 2;
}
}else if( z[3]=='s' || z[3]=='t' ){
stem(&z, "noi", "", m_gt_1);
}
break;
case 's':
if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
case 't':
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
z += 3;
}
break;
case 'v':
case 'z':
if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
}
/* Step 5a */
if( z[0]=='e' ){
if( m_gt_1(z+1) ){
z++;
}else if( m_eq_1(z+1) && !star_oh(z+1) ){
z++;
}
}
/* Step 5b */
if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
z++;
}
/* z[] is now the stemmed word in reverse order. Flip it back
** around into forward order and return.
*/
*pnOut = i = (int)strlen(z);
zOut[i] = 0;
while( *z ){
zOut[--i] = *(z++);
}
}
/*
** Characters that can be part of a token. We assume any character
** whose value is greater than 0x80 (any UTF character) can be
** part of a token. In other words, delimiters all must have
** values of 0x7f or lower.
*/
static const char porterIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to porterOpen().
*/
static int porterNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
const char **pzToken, /* OUT: *pzToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
const char *z = c->zInput;
while( c->iOffset<c->nInput ){
int iStartOffset, ch;
/* Scan past delimiter characters */
while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int n = c->iOffset-iStartOffset;
if( n>c->nAllocated ){
char *pNew;
c->nAllocated = n+20;
pNew = sqlite3_realloc(c->zToken, c->nAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->zToken = pNew;
}
porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
*pzToken = c->zToken;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the porter-stemmer tokenizer
*/
static const sqlite3_tokenizer_module porterTokenizerModule = {
0,
porterCreate,
porterDestroy,
porterOpen,
porterClose,
porterNext,
};
/*
** Allocate a new porter tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts3PorterTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &porterTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
File diff suppressed because it is too large Load Diff
-360
View File
@@ -1,360 +0,0 @@
/*
** 2011 Jan 27
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file is not part of the production FTS code. It is only used for
** testing. It contains a virtual table implementation that provides direct
** access to the full-text index of an FTS table.
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_TEST
#include "fts3Int.h"
#include <string.h>
#include <assert.h>
typedef struct Fts3termTable Fts3termTable;
typedef struct Fts3termCursor Fts3termCursor;
struct Fts3termTable {
sqlite3_vtab base; /* Base class used by SQLite core */
Fts3Table *pFts3Tab;
};
struct Fts3termCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3SegFilter filter;
int isEof; /* True if cursor is at EOF */
char *pNext;
sqlite3_int64 iRowid; /* Current 'rowid' value */
sqlite3_int64 iDocid; /* Current 'docid' value */
int iCol; /* Current 'col' value */
int iPos; /* Current 'pos' value */
};
/*
** Schema of the terms table.
*/
#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, docid, col, pos)"
/*
** This function does all the work for both the xConnect and xCreate methods.
** These tables have no persistent representation of their own, so xConnect
** and xCreate are identical operations.
*/
static int fts3termConnectMethod(
sqlite3 *db, /* Database connection */
void *pUnused, /* Unused */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
char const *zDb; /* Name of database (e.g. "main") */
char const *zFts3; /* Name of fts3 table */
int nDb; /* Result of strlen(zDb) */
int nFts3; /* Result of strlen(zFts3) */
int nByte; /* Bytes of space to allocate here */
int rc; /* value returned by declare_vtab() */
Fts3termTable *p; /* Virtual table object to return */
UNUSED_PARAMETER(pUnused);
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
*pzErr = sqlite3_mprintf(
"wrong number of arguments to fts4term constructor"
);
return SQLITE_ERROR;
}
zDb = argv[1];
nDb = strlen(zDb);
zFts3 = argv[3];
nFts3 = strlen(zFts3);
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
p = (Fts3termTable *)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, nByte);
p->pFts3Tab = (Fts3Table *)&p[1];
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
*ppVtab = (sqlite3_vtab *)p;
return SQLITE_OK;
}
/*
** This function does the work for both the xDisconnect and xDestroy methods.
** These tables have no persistent representation of their own, so xDisconnect
** and xDestroy are identical operations.
*/
static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
Fts3termTable *p = (Fts3termTable *)pVtab;
Fts3Table *pFts3 = p->pFts3Tab;
int i;
/* Free any prepared statements held */
for(i=0; i<SizeofArray(pFts3->aStmt); i++){
sqlite3_finalize(pFts3->aStmt[i]);
}
sqlite3_free(pFts3->zSegmentsTbl);
sqlite3_free(p);
return SQLITE_OK;
}
#define FTS4AUX_EQ_CONSTRAINT 1
#define FTS4AUX_GE_CONSTRAINT 2
#define FTS4AUX_LE_CONSTRAINT 4
/*
** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts3termBestIndexMethod(
sqlite3_vtab *pVTab,
sqlite3_index_info *pInfo
){
UNUSED_PARAMETER(pVTab);
/* This vtab naturally does "ORDER BY term, docid, col, pos". */
if( pInfo->nOrderBy ){
int i;
for(i=0; i<pInfo->nOrderBy; i++){
if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break;
}
if( i==pInfo->nOrderBy ){
pInfo->orderByConsumed = 1;
}
}
return SQLITE_OK;
}
/*
** xOpen - Open a cursor.
*/
static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts3termCursor *pCsr; /* Pointer to cursor object to return */
UNUSED_PARAMETER(pVTab);
pCsr = (Fts3termCursor *)sqlite3_malloc(sizeof(Fts3termCursor));
if( !pCsr ) return SQLITE_NOMEM;
memset(pCsr, 0, sizeof(Fts3termCursor));
*ppCsr = (sqlite3_vtab_cursor *)pCsr;
return SQLITE_OK;
}
/*
** xClose - Close a cursor.
*/
static int fts3termCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
sqlite3Fts3SegmentsClose(pFts3);
sqlite3Fts3SegReaderFinish(&pCsr->csr);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** xNext - Advance the cursor to the next row, if any.
*/
static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
int rc;
sqlite3_int64 v;
/* Increment our pretend rowid value. */
pCsr->iRowid++;
/* Advance to the next term in the full-text index. */
if( pCsr->csr.aDoclist==0
|| pCsr->pNext>=&pCsr->csr.aDoclist[pCsr->csr.nDoclist-1]
){
rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
if( rc!=SQLITE_ROW ){
pCsr->isEof = 1;
return rc;
}
pCsr->iCol = 0;
pCsr->iPos = 0;
pCsr->iDocid = 0;
pCsr->pNext = pCsr->csr.aDoclist;
/* Read docid */
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &pCsr->iDocid);
}
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
if( v==0 ){
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
pCsr->iDocid += v;
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
pCsr->iCol = 0;
pCsr->iPos = 0;
}
if( v==1 ){
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
pCsr->iCol += v;
pCsr->iPos = 0;
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
}
pCsr->iPos += (v - 2);
return SQLITE_OK;
}
/*
** xFilter - Initialize a cursor to point at the start of its data.
*/
static int fts3termFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
int rc;
UNUSED_PARAMETER(nVal);
UNUSED_PARAMETER(idxNum);
UNUSED_PARAMETER(idxStr);
UNUSED_PARAMETER(apVal);
assert( idxStr==0 && idxNum==0 );
/* In case this cursor is being reused, close and zero it. */
testcase(pCsr->filter.zTerm);
sqlite3Fts3SegReaderFinish(&pCsr->csr);
memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
);
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
}
if( rc==SQLITE_OK ){
rc = fts3termNextMethod(pCursor);
}
return rc;
}
/*
** xEof - Return true if the cursor is at EOF, or false otherwise.
*/
static int fts3termEofMethod(sqlite3_vtab_cursor *pCursor){
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
return pCsr->isEof;
}
/*
** xColumn - Return a column value.
*/
static int fts3termColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts3termCursor *p = (Fts3termCursor *)pCursor;
assert( iCol>=0 && iCol<=3 );
switch( iCol ){
case 0:
sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
break;
case 1:
sqlite3_result_int64(pCtx, p->iDocid);
break;
case 2:
sqlite3_result_int64(pCtx, p->iCol);
break;
default:
sqlite3_result_int64(pCtx, p->iPos);
break;
}
return SQLITE_OK;
}
/*
** xRowid - Return the current rowid for the cursor.
*/
static int fts3termRowidMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite_int64 *pRowid /* OUT: Rowid value */
){
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
*pRowid = pCsr->iRowid;
return SQLITE_OK;
}
/*
** Register the fts3term module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts3InitTerm(sqlite3 *db){
static const sqlite3_module fts3term_module = {
0, /* iVersion */
fts3termConnectMethod, /* xCreate */
fts3termConnectMethod, /* xConnect */
fts3termBestIndexMethod, /* xBestIndex */
fts3termDisconnectMethod, /* xDisconnect */
fts3termDisconnectMethod, /* xDestroy */
fts3termOpenMethod, /* xOpen */
fts3termCloseMethod, /* xClose */
fts3termFilterMethod, /* xFilter */
fts3termNextMethod, /* xNext */
fts3termEofMethod, /* xEof */
fts3termColumnMethod, /* xColumn */
fts3termRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
0 /* xRename */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0);
return rc;
}
#endif
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-494
View File
@@ -1,494 +0,0 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is part of an SQLite module implementing full-text search.
** This particular file implements the generic tokenizer interface.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS3 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS3 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include "sqlite3ext.h"
#ifndef SQLITE_CORE
SQLITE_EXTENSION_INIT1
#endif
#include "fts3Int.h"
#include <assert.h>
#include <string.h>
/*
** Implementation of the SQL scalar function for accessing the underlying
** hash table. This function may be called as follows:
**
** SELECT <function-name>(<key-name>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
**
** If the <pointer> argument is specified, it must be a blob value
** containing a pointer to be stored as the hash data corresponding
** to the string <key-name>. If <pointer> is not specified, then
** the string <key-name> must already exist in the has table. Otherwise,
** an error is returned.
**
** Whether or not the <pointer> argument is specified, the value returned
** is a blob containing the pointer stored as the hash data corresponding
** to string <key-name> (after the hash-table is updated, if applicable).
*/
static void scalarFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
Fts3Hash *pHash;
void *pPtr = 0;
const unsigned char *zName;
int nName;
assert( argc==1 || argc==2 );
pHash = (Fts3Hash *)sqlite3_user_data(context);
zName = sqlite3_value_text(argv[0]);
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( n!=sizeof(pPtr) ){
sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
pPtr = *(void **)sqlite3_value_blob(argv[1]);
pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
if( pOld==pPtr ){
sqlite3_result_error(context, "out of memory", -1);
return;
}
}else{
pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
if( !pPtr ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
}
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
int sqlite3Fts3IsIdChar(char c){
static const char isFtsIdChar[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
return (c&0x80 || isFtsIdChar[(int)(c)]);
}
const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
const char *z1;
const char *z2 = 0;
/* Find the start of the next token. */
z1 = zStr;
while( z2==0 ){
char c = *z1;
switch( c ){
case '\0': return 0; /* No more tokens here */
case '\'':
case '"':
case '`': {
z2 = z1;
while( *++z2 && (*z2!=c || *++z2==c) );
break;
}
case '[':
z2 = &z1[1];
while( *z2 && z2[0]!=']' ) z2++;
if( *z2 ) z2++;
break;
default:
if( sqlite3Fts3IsIdChar(*z1) ){
z2 = &z1[1];
while( sqlite3Fts3IsIdChar(*z2) ) z2++;
}else{
z1++;
}
}
}
*pn = (int)(z2-z1);
return z1;
}
int sqlite3Fts3InitTokenizer(
Fts3Hash *pHash, /* Tokenizer hash table */
const char *zArg, /* Tokenizer name */
sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
char **pzErr /* OUT: Set to malloced error message */
){
int rc;
char *z = (char *)zArg;
int n;
char *zCopy;
char *zEnd; /* Pointer to nul-term of zCopy */
sqlite3_tokenizer_module *m;
zCopy = sqlite3_mprintf("%s", zArg);
if( !zCopy ) return SQLITE_NOMEM;
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
z[n] = '\0';
sqlite3Fts3Dequote(z);
m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
}else{
char const **aArg = 0;
int iArg = 0;
z = &z[n+1];
while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
int nNew = sizeof(char *)*(iArg+1);
char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
if( !aNew ){
sqlite3_free(zCopy);
sqlite3_free((void *)aArg);
return SQLITE_NOMEM;
}
aArg = aNew;
aArg[iArg++] = z;
z[n] = '\0';
sqlite3Fts3Dequote(z);
z = &z[n+1];
}
rc = m->xCreate(iArg, aArg, ppTok);
assert( rc!=SQLITE_OK || *ppTok );
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("unknown tokenizer");
}else{
(*ppTok)->pModule = m;
}
sqlite3_free((void *)aArg);
}
sqlite3_free(zCopy);
return rc;
}
#ifdef SQLITE_TEST
#include <tcl.h>
#include <string.h>
/*
** Implementation of a special SQL scalar function for testing tokenizers
** designed to be used in concert with the Tcl testing framework. This
** function must be called with two arguments:
**
** SELECT <function-name>(<key-name>, <input-string>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
**
** The return value is a string that may be interpreted as a Tcl
** list. For each token in the <input-string>, three elements are
** added to the returned list. The first is the token position, the
** second is the token text (folded, stemmed, etc.) and the third is the
** substring of <input-string> associated with the token. For example,
** using the built-in "simple" tokenizer:
**
** SELECT fts_tokenizer_test('simple', 'I don't see how');
**
** will return the string:
**
** "{0 i I 1 dont don't 2 see see 3 how how}"
**
*/
static void testFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
Fts3Hash *pHash;
sqlite3_tokenizer_module *p;
sqlite3_tokenizer *pTokenizer = 0;
sqlite3_tokenizer_cursor *pCsr = 0;
const char *zErr = 0;
const char *zName;
int nName;
const char *zInput;
int nInput;
const char *zArg = 0;
const char *zToken;
int nToken;
int iStart;
int iEnd;
int iPos;
Tcl_Obj *pRet;
assert( argc==2 || argc==3 );
nName = sqlite3_value_bytes(argv[0]);
zName = (const char *)sqlite3_value_text(argv[0]);
nInput = sqlite3_value_bytes(argv[argc-1]);
zInput = (const char *)sqlite3_value_text(argv[argc-1]);
if( argc==3 ){
zArg = (const char *)sqlite3_value_text(argv[1]);
}
pHash = (Fts3Hash *)sqlite3_user_data(context);
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
zErr = "error in xCreate()";
goto finish;
}
pTokenizer->pModule = p;
if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
zErr = "error in xOpen()";
goto finish;
}
pCsr->pTokenizer = pTokenizer;
while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
zToken = &zInput[iStart];
nToken = iEnd-iStart;
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
}
if( SQLITE_OK!=p->xClose(pCsr) ){
zErr = "error in xClose()";
goto finish;
}
if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
zErr = "error in xDestroy()";
goto finish;
}
finish:
if( zErr ){
sqlite3_result_error(context, zErr, -1);
}else{
sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
}
Tcl_DecrRefCount(pRet);
}
static
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
static
int queryTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
/*
** Implementation of the scalar function fts3_tokenizer_internal_test().
** This function is used for testing only, it is not included in the
** build unless SQLITE_TEST is defined.
**
** The purpose of this is to test that the fts3_tokenizer() function
** can be used as designed by the C-code in the queryTokenizer and
** registerTokenizer() functions above. These two functions are repeated
** in the README.tokenizer file as an example, so it is important to
** test them.
**
** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
** function with no arguments. An assert() will fail if a problem is
** detected. i.e.:
**
** SELECT fts3_tokenizer_internal_test();
**
*/
static void intTestFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int rc;
const sqlite3_tokenizer_module *p1;
const sqlite3_tokenizer_module *p2;
sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
UNUSED_PARAMETER(argc);
UNUSED_PARAMETER(argv);
/* Test the query function */
sqlite3Fts3SimpleTokenizerModule(&p1);
rc = queryTokenizer(db, "simple", &p2);
assert( rc==SQLITE_OK );
assert( p1==p2 );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_ERROR );
assert( p2==0 );
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
#endif
/*
** Set up SQL objects in database db used to access the contents of
** the hash table pointed to by argument pHash. The hash table must
** been initialised to use string keys, and to take a private copy
** of the key when a value is inserted. i.e. by a call similar to:
**
** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
**
** This function adds a scalar function (see header comment above
** scalarFunc() in this file for details) and, if ENABLE_TABLE is
** defined at compilation time, a temporary virtual table (see header
** comment above struct HashTableVtab) to the database schema. Both
** provide read/write access to the contents of *pHash.
**
** The third argument to this function, zName, is used as the name
** of both the scalar and, if created, the virtual table.
*/
int sqlite3Fts3InitHashTable(
sqlite3 *db,
Fts3Hash *pHash,
const char *zName
){
int rc = SQLITE_OK;
void *p = (void *)pHash;
const int any = SQLITE_ANY;
#ifdef SQLITE_TEST
char *zTest = 0;
char *zTest2 = 0;
void *pdb = (void *)db;
zTest = sqlite3_mprintf("%s_test", zName);
zTest2 = sqlite3_mprintf("%s_internal_test", zName);
if( !zTest || !zTest2 ){
rc = SQLITE_NOMEM;
}
#endif
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
}
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
}
#ifdef SQLITE_TEST
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0);
}
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0);
}
if( SQLITE_OK==rc ){
rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
}
#endif
#ifdef SQLITE_TEST
sqlite3_free(zTest);
sqlite3_free(zTest2);
#endif
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-152
View File
@@ -1,152 +0,0 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _FTS3_TOKENIZER_H_
#define _FTS3_TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface. When a new tokenizer
** implementation is registered, the caller provides a pointer to
** an sqlite3_tokenizer_module containing pointers to the callback
** functions that make up an implementation.
**
** When an fts3 table is created, it passes any arguments passed to
** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
** implementation. The xCreate() function in turn returns an
** sqlite3_tokenizer structure representing the specific tokenizer to
** be used for the fts3 table (customized by the tokenizer clause arguments).
**
** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
** method is called. It returns an sqlite3_tokenizer_cursor object
** that may be used to tokenize a specific input buffer based on
** the tokenization rules supplied by a specific sqlite3_tokenizer
** object.
*/
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
struct sqlite3_tokenizer_module {
/*
** Structure version. Should always be set to 0.
*/
int iVersion;
/*
** Create a new tokenizer. The values in the argv[] array are the
** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
** TABLE statement that created the fts3 table. For example, if
** the following SQL is executed:
**
** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
**
** then argc is set to 2, and the argv[] array contains pointers
** to the strings "arg1" and "arg2".
**
** This method should return either SQLITE_OK (0), or an SQLite error
** code. If SQLITE_OK is returned, then *ppTokenizer should be set
** to point at the newly created tokenizer structure. The generic
** sqlite3_tokenizer.pModule variable should not be initialised by
** this callback. The caller will do so.
*/
int (*xCreate)(
int argc, /* Size of argv array */
const char *const*argv, /* Tokenizer argument strings */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
);
/*
** Destroy an existing tokenizer. The fts3 module calls this method
** exactly once for each successful call to xCreate().
*/
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Create a tokenizer cursor to tokenize an input buffer. The caller
** is responsible for ensuring that the input buffer remains valid
** until the cursor is closed (using the xClose() method).
*/
int (*xOpen)(
sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
const char *pInput, int nBytes, /* Input buffer */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
);
/*
** Destroy an existing tokenizer cursor. The fts3 module calls this
** method exactly once for each successful call to xOpen().
*/
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
/*
** Retrieve the next token from the tokenizer cursor pCursor. This
** method should either return SQLITE_OK and set the values of the
** "OUT" variables identified below, or SQLITE_DONE to indicate that
** the end of the buffer has been reached, or an SQLite error code.
**
** *ppToken should be set to point at a buffer containing the
** normalized version of the token (i.e. after any case-folding and/or
** stemming has been performed). *pnBytes should be set to the length
** of this buffer in bytes. The input text that generated the token is
** identified by the byte offsets returned in *piStartOffset and
** *piEndOffset. *piStartOffset should be set to the index of the first
** byte of the token in the input buffer. *piEndOffset should be set
** to the index of the first byte just past the end of the token in
** the input buffer.
**
** The buffer *ppToken is set to point at is managed by the tokenizer
** implementation. It is only required to be valid until the next call
** to xNext() or xClose().
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xNext)(
sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
int *piStartOffset, /* OUT: Byte offset of token in input buffer */
int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
int *piPosition /* OUT: Number of tokens returned before this one */
);
};
struct sqlite3_tokenizer {
const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
int fts3_global_term_cnt(int iTerm, int iCol);
int fts3_term_cnt(int iTerm, int iCol);
#endif /* _FTS3_TOKENIZER_H_ */
-234
View File
@@ -1,234 +0,0 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Implementation of the "simple" full-text-search tokenizer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS3 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS3 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include "fts3Int.h"
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fts3_tokenizer.h"
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
char delim[128]; /* flag ASCII delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
int iOffset; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *pToken; /* storage for current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
static int simpleDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
static int fts3_isalnum(int x){
return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
}
/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
int i, n = (int)strlen(argv[1]);
for(i=0; i<n; i++){
unsigned char ch = argv[1][i];
/* We explicitly don't support UTF-8 delimiters for now. */
if( ch>=0x80 ){
sqlite3_free(t);
return SQLITE_ERROR;
}
t->delim[ch] = 1;
}
} else {
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
}
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int simpleOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *pInput, int nBytes, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
simple_tokenizer_cursor *c;
UNUSED_PARAMETER(pTokenizer);
c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->pInput = pInput;
if( pInput==0 ){
c->nBytes = 0;
}else if( nBytes<0 ){
c->nBytes = (int)strlen(pInput);
}else{
c->nBytes = nBytes;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->pToken = NULL; /* no space allocated, yet. */
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
sqlite3_free(c->pToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
unsigned char *p = (unsigned char *)c->pInput;
while( c->iOffset<c->nBytes ){
int iStartOffset;
/* Scan past delimiter characters */
while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int i, n = c->iOffset-iStartOffset;
if( n>c->nTokenAllocated ){
char *pNew;
c->nTokenAllocated = n+20;
pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->pToken = pNew;
}
for(i=0; i<n; i++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
}
*ppToken = c->pToken;
*pnBytes = n;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
/*
** Allocate a new simple tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts3SimpleTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &simpleTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
File diff suppressed because it is too large Load Diff
-122
View File
@@ -1,122 +0,0 @@
#--------------------------------------------------------------------------
# This script contains several sub-programs used to test FTS3/FTS4
# performance. It does not run the queries directly, but generates SQL
# scripts that can be run using the shell tool.
#
# The following cases are tested:
#
# 1. Inserting documents into an FTS3 table.
# 2. Optimizing an FTS3 table (i.e. "INSERT INTO t1 VALUES('optimize')").
# 3. Deleting documents from an FTS3 table.
# 4. Querying FTS3 tables.
#
# Number of tokens in vocabulary. And number of tokens in each document.
#
set VOCAB_SIZE 2000
set DOC_SIZE 100
set NUM_INSERTS 100000
set NUM_SELECTS 1000
# Force everything in this script to be deterministic.
#
expr {srand(0)}
proc usage {} {
puts stderr "Usage: $::argv0 <rows> <selects>"
exit -1
}
proc sql {sql} {
puts $::fd $sql
}
# Return a list of $nWord randomly generated tokens each between 2 and 10
# characters in length.
#
proc build_vocab {nWord} {
set ret [list]
set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
for {set i 0} {$i<$nWord} {incr i} {
set len [expr {int((rand()*9.0)+2)}]
set term ""
for {set j 0} {$j<$len} {incr j} {
append term [lindex $chars [expr {int(rand()*[llength $chars])}]]
}
lappend ret $term
}
set ret
}
proc select_term {} {
set n [llength $::vocab]
set t [expr int(rand()*$n*3)]
if {$t>=2*$n} { set t [expr {($t-2*$n)/100}] }
if {$t>=$n} { set t [expr {($t-$n)/10}] }
lindex $::vocab $t
}
proc select_doc {nTerm} {
set ret [list]
for {set i 0} {$i<$nTerm} {incr i} {
lappend ret [select_term]
}
set ret
}
proc test_1 {nInsert} {
sql "PRAGMA synchronous = OFF;"
sql "DROP TABLE IF EXISTS t1;"
sql "CREATE VIRTUAL TABLE t1 USING fts4;"
for {set i 0} {$i < $nInsert} {incr i} {
set doc [select_doc $::DOC_SIZE]
sql "INSERT INTO t1 VALUES('$doc');"
}
}
proc test_2 {} {
sql "INSERT INTO t1(t1) VALUES('optimize');"
}
proc test_3 {nSelect} {
for {set i 0} {$i < $nSelect} {incr i} {
sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term]';"
}
}
proc test_4 {nSelect} {
for {set i 0} {$i < $nSelect} {incr i} {
sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term] [select_term]';"
}
}
if {[llength $argv]!=0} usage
set ::vocab [build_vocab $::VOCAB_SIZE]
set ::fd [open fts3speed_insert.sql w]
test_1 $NUM_INSERTS
close $::fd
set ::fd [open fts3speed_select.sql w]
test_3 $NUM_SELECTS
close $::fd
set ::fd [open fts3speed_select2.sql w]
test_4 $NUM_SELECTS
close $::fd
set ::fd [open fts3speed_optimize.sql w]
test_2
close $::fd
puts "Success. Created files:"
puts " fts3speed_insert.sql"
puts " fts3speed_select.sql"
puts " fts3speed_select2.sql"
puts " fts3speed_optimize.sql"
-115
View File
@@ -1,115 +0,0 @@
#!/usr/bin/tclsh
#
# This script builds a single C code file holding all of FTS3 code.
# The name of the output file is fts3amal.c. To build this file,
# first do:
#
# make target_source
#
# The make target above moves all of the source code files into
# a subdirectory named "tsrc". (This script expects to find the files
# there and will not work if they are not found.)
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
# tclsh mkfts3amal.tcl
#
# The amalgamated FTS3 code will be written into fts3amal.c
#
# Open the output file and write a header comment at the beginning
# of the file.
#
set out [open fts3amal.c w]
set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
puts $out [subst \
{/******************************************************************************
** This file is an amalgamation of separate C source files from the SQLite
** Full Text Search extension 2 (fts3). By combining all the individual C
** code files into this single large file, the entire code can be compiled
** as a one translation unit. This allows many compilers to do optimizations
** that would not be possible if the files were compiled separately. It also
** makes the code easier to import into other projects.
**
** This amalgamation was generated on $today.
*/}]
# These are the header files used by FTS3. The first time any of these
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line. The file only needs to be included once.
#
foreach hdr {
fts3.h
fts3_hash.h
fts3_tokenizer.h
sqlite3.h
sqlite3ext.h
} {
set available_hdr($hdr) 1
}
# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}
# Insert a comment into the code
#
proc section_comment {text} {
global out s78
set n [string length $text]
set nstar [expr {60 - $n}]
set stars [string range $s78 0 $nstar]
puts $out "/************** $text $stars/"
}
# Read the source file named $filename and write it into the
# sqlite3.c output file. If any #include statements are seen,
# process them approprately.
#
proc copy_file {filename} {
global seen_hdr available_hdr out
set tail [file tail $filename]
section_comment "Begin file $tail"
set in [open $filename r]
while {![eof $in]} {
set line [gets $in]
if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
if {[info exists available_hdr($hdr)]} {
if {$available_hdr($hdr)} {
section_comment "Include $hdr in the middle of $tail"
copy_file tsrc/$hdr
section_comment "Continuing where we left off in $tail"
}
} elseif {![info exists seen_hdr($hdr)]} {
set seen_hdr($hdr) 1
puts $out $line
}
} elseif {[regexp {^#ifdef __cplusplus} $line]} {
puts $out "#if 0"
} elseif {[regexp {^#line} $line]} {
# Skip #line directives.
} else {
puts $out $line
}
}
close $in
section_comment "End of $tail"
}
# Process the source files. Process files containing commonly
# used subroutines first in order to help the compiler find
# inlining opportunities.
#
foreach file {
fts3.c
fts3_hash.c
fts3_porter.c
fts3_tokenizer.c
fts3_tokenizer1.c
} {
copy_file tsrc/$file
}
close $out
-169
View File
@@ -1,169 +0,0 @@
This directory contains source code for the SQLite "ICU" extension, an
integration of the "International Components for Unicode" library with
SQLite. Documentation follows.
1. Features
1.1 SQL Scalars upper() and lower()
1.2 Unicode Aware LIKE Operator
1.3 ICU Collation Sequences
1.4 SQL REGEXP Operator
2. Compilation and Usage
3. Bugs, Problems and Security Issues
3.1 The "case_sensitive_like" Pragma
3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro
3.3 Collation Sequence Security Issue
1. FEATURES
1.1 SQL Scalars upper() and lower()
SQLite's built-in implementations of these two functions only
provide case mapping for the 26 letters used in the English
language. The ICU based functions provided by this extension
provide case mapping, where defined, for the full range of
unicode characters.
ICU provides two types of case mapping, "general" case mapping and
"language specific". Refer to ICU documentation for the differences
between the two. Specifically:
http://www.icu-project.org/userguide/caseMappings.html
http://www.icu-project.org/userguide/posix.html#case_mappings
To utilise "general" case mapping, the upper() or lower() scalar
functions are invoked with one argument:
upper('ABC') -> 'abc'
lower('abc') -> 'ABC'
To access ICU "language specific" case mapping, upper() or lower()
should be invoked with two arguments. The second argument is the name
of the locale to use. Passing an empty string ("") or SQL NULL value
as the second argument is the same as invoking the 1 argument version
of upper() or lower():
lower('I', 'en_us') -> 'i'
lower('I', 'tr_tr') -> 'ı' (small dotless i)
1.2 Unicode Aware LIKE Operator
Similarly to the upper() and lower() functions, the built-in SQLite LIKE
operator understands case equivalence for the 26 letters of the English
language alphabet. The implementation of LIKE included in this
extension uses the ICU function u_foldCase() to provide case
independent comparisons for the full range of unicode characters.
The U_FOLD_CASE_DEFAULT flag is passed to u_foldCase(), meaning the
dotless 'I' character used in the Turkish language is considered
to be in the same equivalence class as the dotted 'I' character
used by many languages (including English).
1.3 ICU Collation Sequences
A special SQL scalar function, icu_load_collation() is provided that
may be used to register ICU collation sequences with SQLite. It
is always called with exactly two arguments, the ICU locale
identifying the collation sequence to ICU, and the name of the
SQLite collation sequence to create. For example, to create an
SQLite collation sequence named "turkish" using Turkish language
sorting rules, the SQL statement:
SELECT icu_load_collation('tr_TR', 'turkish');
Or, for Australian English:
SELECT icu_load_collation('en_AU', 'australian');
The identifiers "turkish" and "australian" may then be used
as collation sequence identifiers in SQL statements:
CREATE TABLE aust_turkish_penpals(
australian_penpal_name TEXT COLLATE australian,
turkish_penpal_name TEXT COLLATE turkish
);
1.4 SQL REGEXP Operator
This extension provides an implementation of the SQL binary
comparision operator "REGEXP", based on the regular expression functions
provided by the ICU library. The syntax of the operator is as described
in SQLite documentation:
<string> REGEXP <re-pattern>
This extension uses the ICU defaults for regular expression matching
behaviour. Specifically, this means that:
* Matching is case-sensitive,
* Regular expression comments are not allowed within patterns, and
* The '^' and '$' characters match the beginning and end of the
<string> argument, not the beginning and end of lines within
the <string> argument.
Even more specifically, the value passed to the "flags" parameter
of ICU C function uregex_open() is 0.
2 COMPILATION AND USAGE
The easiest way to compile and use the ICU extension is to build
and use it as a dynamically loadable SQLite extension. To do this
using gcc on *nix:
gcc -shared icu.c `icu-config --ldflags` -o libSqliteIcu.so
You may need to add "-I" flags so that gcc can find sqlite3ext.h
and sqlite3.h. The resulting shared lib, libSqliteIcu.so, may be
loaded into sqlite in the same way as any other dynamically loadable
extension.
3 BUGS, PROBLEMS AND SECURITY ISSUES
3.1 The "case_sensitive_like" Pragma
This extension does not work well with the "case_sensitive_like"
pragma. If this pragma is used before the ICU extension is loaded,
then the pragma has no effect. If the pragma is used after the ICU
extension is loaded, then SQLite ignores the ICU implementation and
always uses the built-in LIKE operator.
The ICU extension LIKE operator is always case insensitive.
3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro
Passing very long patterns to the built-in SQLite LIKE operator can
cause excessive CPU usage. To curb this problem, SQLite defines the
SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a
pattern in bytes (irrespective of encoding). The default value is
defined in internal header file "limits.h".
The ICU extension LIKE implementation suffers from the same
problem and uses the same solution. However, since the ICU extension
code does not include the SQLite file "limits.h", modifying
the default value therein does not affect the ICU extension.
The default value of SQLITE_MAX_LIKE_PATTERN_LENGTH used by
the ICU extension LIKE operator is 50000, defined in source
file "icu.c".
3.3 Collation Sequence Security Issue
Internally, SQLite assumes that indices stored in database files
are sorted according to the collation sequence indicated by the
SQL schema. Changing the definition of a collation sequence after
an index has been built is therefore equivalent to database
corruption. The SQLite library is not very well tested under
these conditions, and may contain potential buffer overruns
or other programming errors that could be exploited by a malicious
programmer.
If the ICU extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they
should be prevented from invoking the icu_load_collation() function,
possibly using the authorisation callback.
-501
View File
@@ -1,501 +0,0 @@
/*
** 2007 May 6
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $
**
** This file implements an integration between the ICU library
** ("International Components for Unicode", an open-source library
** for handling unicode data) and SQLite. The integration uses
** ICU to provide the following to SQLite:
**
** * An implementation of the SQL regexp() function (and hence REGEXP
** operator) using the ICU uregex_XX() APIs.
**
** * Implementations of the SQL scalar upper() and lower() functions
** for case mapping.
**
** * Integration of ICU and SQLite collation seqences.
**
** * An implementation of the LIKE operator that uses ICU to
** provide case-independent matching.
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
/* Include ICU headers */
#include <unicode/utypes.h>
#include <unicode/uregex.h>
#include <unicode/ustring.h>
#include <unicode/ucol.h>
#include <assert.h>
#ifndef SQLITE_CORE
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#else
#include "sqlite3.h"
#endif
/*
** Maximum length (in bytes) of the pattern in a LIKE or GLOB
** operator.
*/
#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
#endif
/*
** Version of sqlite3_free() that is always a function, never a macro.
*/
static void xFree(void *p){
sqlite3_free(p);
}
/*
** Compare two UTF-8 strings for equality where the first string is
** a "LIKE" expression. Return true (1) if they are the same and
** false (0) if they are different.
*/
static int icuLikeCompare(
const uint8_t *zPattern, /* LIKE pattern */
const uint8_t *zString, /* The UTF-8 string to compare against */
const UChar32 uEsc /* The escape character */
){
static const int MATCH_ONE = (UChar32)'_';
static const int MATCH_ALL = (UChar32)'%';
int iPattern = 0; /* Current byte index in zPattern */
int iString = 0; /* Current byte index in zString */
int prevEscape = 0; /* True if the previous character was uEsc */
while( zPattern[iPattern]!=0 ){
/* Read (and consume) the next character from the input pattern. */
UChar32 uPattern;
U8_NEXT_UNSAFE(zPattern, iPattern, uPattern);
assert(uPattern!=0);
/* There are now 4 possibilities:
**
** 1. uPattern is an unescaped match-all character "%",
** 2. uPattern is an unescaped match-one character "_",
** 3. uPattern is an unescaped escape character, or
** 4. uPattern is to be handled as an ordinary character
*/
if( !prevEscape && uPattern==MATCH_ALL ){
/* Case 1. */
uint8_t c;
/* Skip any MATCH_ALL or MATCH_ONE characters that follow a
** MATCH_ALL. For each MATCH_ONE, skip one character in the
** test string.
*/
while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){
if( c==MATCH_ONE ){
if( zString[iString]==0 ) return 0;
U8_FWD_1_UNSAFE(zString, iString);
}
iPattern++;
}
if( zPattern[iPattern]==0 ) return 1;
while( zString[iString] ){
if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){
return 1;
}
U8_FWD_1_UNSAFE(zString, iString);
}
return 0;
}else if( !prevEscape && uPattern==MATCH_ONE ){
/* Case 2. */
if( zString[iString]==0 ) return 0;
U8_FWD_1_UNSAFE(zString, iString);
}else if( !prevEscape && uPattern==uEsc){
/* Case 3. */
prevEscape = 1;
}else{
/* Case 4. */
UChar32 uString;
U8_NEXT_UNSAFE(zString, iString, uString);
uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT);
uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT);
if( uString!=uPattern ){
return 0;
}
prevEscape = 0;
}
}
return zString[iString]==0;
}
/*
** Implementation of the like() SQL function. This function implements
** the build-in LIKE operator. The first argument to the function is the
** pattern and the second argument is the string. So, the SQL statements:
**
** A LIKE B
**
** is implemented as like(B, A). If there is an escape character E,
**
** A LIKE B ESCAPE E
**
** is mapped to like(B, A, E).
*/
static void icuLikeFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
const unsigned char *zA = sqlite3_value_text(argv[0]);
const unsigned char *zB = sqlite3_value_text(argv[1]);
UChar32 uEsc = 0;
/* Limit the length of the LIKE or GLOB pattern to avoid problems
** of deep recursion and N*N behavior in patternCompare().
*/
if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){
sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1);
return;
}
if( argc==3 ){
/* The escape character string must consist of a single UTF-8 character.
** Otherwise, return an error.
*/
int nE= sqlite3_value_bytes(argv[2]);
const unsigned char *zE = sqlite3_value_text(argv[2]);
int i = 0;
if( zE==0 ) return;
U8_NEXT(zE, i, nE, uEsc);
if( i!=nE){
sqlite3_result_error(context,
"ESCAPE expression must be a single character", -1);
return;
}
}
if( zA && zB ){
sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
}
}
/*
** This function is called when an ICU function called from within
** the implementation of an SQL scalar function returns an error.
**
** The scalar function context passed as the first argument is
** loaded with an error message based on the following two args.
*/
static void icuFunctionError(
sqlite3_context *pCtx, /* SQLite scalar function context */
const char *zName, /* Name of ICU function that failed */
UErrorCode e /* Error code returned by ICU function */
){
char zBuf[128];
sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
zBuf[127] = '\0';
sqlite3_result_error(pCtx, zBuf, -1);
}
/*
** Function to delete compiled regexp objects. Registered as
** a destructor function with sqlite3_set_auxdata().
*/
static void icuRegexpDelete(void *p){
URegularExpression *pExpr = (URegularExpression *)p;
uregex_close(pExpr);
}
/*
** Implementation of SQLite REGEXP operator. This scalar function takes
** two arguments. The first is a regular expression pattern to compile
** the second is a string to match against that pattern. If either
** argument is an SQL NULL, then NULL Is returned. Otherwise, the result
** is 1 if the string matches the pattern, or 0 otherwise.
**
** SQLite maps the regexp() function to the regexp() operator such
** that the following two are equivalent:
**
** zString REGEXP zPattern
** regexp(zPattern, zString)
**
** Uses the following ICU regexp APIs:
**
** uregex_open()
** uregex_matches()
** uregex_close()
*/
static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){
UErrorCode status = U_ZERO_ERROR;
URegularExpression *pExpr;
UBool res;
const UChar *zString = sqlite3_value_text16(apArg[1]);
(void)nArg; /* Unused parameter */
/* If the left hand side of the regexp operator is NULL,
** then the result is also NULL.
*/
if( !zString ){
return;
}
pExpr = sqlite3_get_auxdata(p, 0);
if( !pExpr ){
const UChar *zPattern = sqlite3_value_text16(apArg[0]);
if( !zPattern ){
return;
}
pExpr = uregex_open(zPattern, -1, 0, 0, &status);
if( U_SUCCESS(status) ){
sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete);
}else{
assert(!pExpr);
icuFunctionError(p, "uregex_open", status);
return;
}
}
/* Configure the text that the regular expression operates on. */
uregex_setText(pExpr, zString, -1, &status);
if( !U_SUCCESS(status) ){
icuFunctionError(p, "uregex_setText", status);
return;
}
/* Attempt the match */
res = uregex_matches(pExpr, 0, &status);
if( !U_SUCCESS(status) ){
icuFunctionError(p, "uregex_matches", status);
return;
}
/* Set the text that the regular expression operates on to a NULL
** pointer. This is not really necessary, but it is tidier than
** leaving the regular expression object configured with an invalid
** pointer after this function returns.
*/
uregex_setText(pExpr, 0, 0, &status);
/* Return 1 or 0. */
sqlite3_result_int(p, res ? 1 : 0);
}
/*
** Implementations of scalar functions for case mapping - upper() and
** lower(). Function upper() converts its input to upper-case (ABC).
** Function lower() converts to lower-case (abc).
**
** ICU provides two types of case mapping, "general" case mapping and
** "language specific". Refer to ICU documentation for the differences
** between the two.
**
** To utilise "general" case mapping, the upper() or lower() scalar
** functions are invoked with one argument:
**
** upper('ABC') -> 'abc'
** lower('abc') -> 'ABC'
**
** To access ICU "language specific" case mapping, upper() or lower()
** should be invoked with two arguments. The second argument is the name
** of the locale to use. Passing an empty string ("") or SQL NULL value
** as the second argument is the same as invoking the 1 argument version
** of upper() or lower().
**
** lower('I', 'en_us') -> 'i'
** lower('I', 'tr_tr') -> 'ı' (small dotless i)
**
** http://www.icu-project.org/userguide/posix.html#case_mappings
*/
static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
const UChar *zInput;
UChar *zOutput;
int nInput;
int nOutput;
UErrorCode status = U_ZERO_ERROR;
const char *zLocale = 0;
assert(nArg==1 || nArg==2);
if( nArg==2 ){
zLocale = (const char *)sqlite3_value_text(apArg[1]);
}
zInput = sqlite3_value_text16(apArg[0]);
if( !zInput ){
return;
}
nInput = sqlite3_value_bytes16(apArg[0]);
nOutput = nInput * 2 + 2;
zOutput = sqlite3_malloc(nOutput);
if( !zOutput ){
return;
}
if( sqlite3_user_data(p) ){
u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status);
}else{
u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status);
}
if( !U_SUCCESS(status) ){
icuFunctionError(p, "u_strToLower()/u_strToUpper", status);
return;
}
sqlite3_result_text16(p, zOutput, -1, xFree);
}
/*
** Collation sequence destructor function. The pCtx argument points to
** a UCollator structure previously allocated using ucol_open().
*/
static void icuCollationDel(void *pCtx){
UCollator *p = (UCollator *)pCtx;
ucol_close(p);
}
/*
** Collation sequence comparison function. The pCtx argument points to
** a UCollator structure previously allocated using ucol_open().
*/
static int icuCollationColl(
void *pCtx,
int nLeft,
const void *zLeft,
int nRight,
const void *zRight
){
UCollationResult res;
UCollator *p = (UCollator *)pCtx;
res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2);
switch( res ){
case UCOL_LESS: return -1;
case UCOL_GREATER: return +1;
case UCOL_EQUAL: return 0;
}
assert(!"Unexpected return value from ucol_strcoll()");
return 0;
}
/*
** Implementation of the scalar function icu_load_collation().
**
** This scalar function is used to add ICU collation based collation
** types to an SQLite database connection. It is intended to be called
** as follows:
**
** SELECT icu_load_collation(<locale>, <collation-name>);
**
** Where <locale> is a string containing an ICU locale identifier (i.e.
** "en_AU", "tr_TR" etc.) and <collation-name> is the name of the
** collation sequence to create.
*/
static void icuLoadCollation(
sqlite3_context *p,
int nArg,
sqlite3_value **apArg
){
sqlite3 *db = (sqlite3 *)sqlite3_user_data(p);
UErrorCode status = U_ZERO_ERROR;
const char *zLocale; /* Locale identifier - (eg. "jp_JP") */
const char *zName; /* SQL Collation sequence name (eg. "japanese") */
UCollator *pUCollator; /* ICU library collation object */
int rc; /* Return code from sqlite3_create_collation_x() */
assert(nArg==2);
zLocale = (const char *)sqlite3_value_text(apArg[0]);
zName = (const char *)sqlite3_value_text(apArg[1]);
if( !zLocale || !zName ){
return;
}
pUCollator = ucol_open(zLocale, &status);
if( !U_SUCCESS(status) ){
icuFunctionError(p, "ucol_open", status);
return;
}
assert(p);
rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator,
icuCollationColl, icuCollationDel
);
if( rc!=SQLITE_OK ){
ucol_close(pUCollator);
sqlite3_result_error(p, "Error registering collation function", -1);
}
}
/*
** Register the ICU extension functions with database db.
*/
int sqlite3IcuInit(sqlite3 *db){
struct IcuScalar {
const char *zName; /* Function name */
int nArg; /* Number of arguments */
int enc; /* Optimal text encoding */
void *pContext; /* sqlite3_user_data() context */
void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
} scalars[] = {
{"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc},
{"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16},
{"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16},
{"like", 2, SQLITE_UTF8, 0, icuLikeFunc},
{"like", 3, SQLITE_UTF8, 0, icuLikeFunc},
{"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation},
};
int rc = SQLITE_OK;
int i;
for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
struct IcuScalar *p = &scalars[i];
rc = sqlite3_create_function(
db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0
);
}
return rc;
}
#if !SQLITE_CORE
int sqlite3_extension_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
SQLITE_EXTENSION_INIT2(pApi)
return sqlite3IcuInit(db);
}
#endif
#endif
-27
View File
@@ -1,27 +0,0 @@
/*
** 2008 May 26
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This header file is used by programs that want to link against the
** ICU extension. All it does is declare the sqlite3IcuInit() interface.
*/
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3IcuInit(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-120
View File
@@ -1,120 +0,0 @@
This directory contains an SQLite extension that implements a virtual
table type that allows users to create, query and manipulate r-tree[1]
data structures inside of SQLite databases. Users create, populate
and query r-tree structures using ordinary SQL statements.
1. SQL Interface
1.1 Table Creation
1.2 Data Manipulation
1.3 Data Querying
1.4 Introspection and Analysis
2. Compilation and Deployment
3. References
1. SQL INTERFACE
1.1 Table Creation.
All r-tree virtual tables have an odd number of columns between
3 and 11. Unlike regular SQLite tables, r-tree tables are strongly
typed.
The leftmost column is always the pimary key and contains 64-bit
integer values. Each subsequent column contains a 32-bit real
value. For each pair of real values, the first (leftmost) must be
less than or equal to the second. R-tree tables may be
constructed using the following syntax:
CREATE VIRTUAL TABLE <name> USING rtree(<column-names>)
For example:
CREATE VIRTUAL TABLE boxes USING rtree(boxno, xmin, xmax, ymin, ymax);
INSERT INTO boxes VALUES(1, 1.0, 3.0, 2.0, 4.0);
Constructing a virtual r-tree table <name> creates the following three
real tables in the database to store the data structure:
<name>_node
<name>_rowid
<name>_parent
Dropping or modifying the contents of these tables directly will
corrupt the r-tree structure. To delete an r-tree from a database,
use a regular DROP TABLE statement:
DROP TABLE <name>;
Dropping the main r-tree table automatically drops the automatically
created tables.
1.2 Data Manipulation (INSERT, UPDATE, DELETE).
The usual INSERT, UPDATE or DELETE syntax is used to manipulate data
stored in an r-tree table. Please note the following:
* Inserting a NULL value into the primary key column has the
same effect as inserting a NULL into an INTEGER PRIMARY KEY
column of a regular table. The system automatically assigns
an unused integer key value to the new record. Usually, this
is one greater than the largest primary key value currently
present in the table.
* Attempting to insert a duplicate primary key value fails with
an SQLITE_CONSTRAINT error.
* Attempting to insert or modify a record such that the value
stored in the (N*2)th column is greater than that stored in
the (N*2+1)th column fails with an SQLITE_CONSTRAINT error.
* When a record is inserted, values are always converted to
the required type (64-bit integer or 32-bit real) as if they
were part of an SQL CAST expression. Non-numeric strings are
converted to zero.
1.3 Queries.
R-tree tables may be queried using all of the same SQL syntax supported
by regular tables. However, some query patterns are more efficient
than others.
R-trees support fast lookup by primary key value (O(logN), like
regular tables).
Any combination of equality and range (<, <=, >, >=) constraints
on spatial data columns may be used to optimize other queries. This
is the key advantage to using r-tree tables instead of creating
indices on regular tables.
1.4 Introspection and Analysis.
TODO: Describe rtreenode() and rtreedepth() functions.
2. COMPILATION AND USAGE
The easiest way to compile and use the RTREE extension is to build
and use it as a dynamically loadable SQLite extension. To do this
using gcc on *nix:
gcc -shared rtree.c -o libSqliteRtree.so
You may need to add "-I" flags so that gcc can find sqlite3ext.h
and sqlite3.h. The resulting shared lib, libSqliteRtree.so, may be
loaded into sqlite in the same way as any other dynamicly loadable
extension.
3. REFERENCES
[1] Atonin Guttman, "R-trees - A Dynamic Index Structure For Spatial
Searching", University of California Berkeley, 1984.
[2] Norbert Beckmann, Hans-Peter Kriegel, Ralf Schneider, Bernhard Seeger,
"The R*-tree: An Efficient and Robust Access Method for Points and
Rectangles", Universitaet Bremen, 1990.
-3277
View File
File diff suppressed because it is too large Load Diff
-26
View File
@@ -1,26 +0,0 @@
/*
** 2008 May 26
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This header file is used by programs that want to link against the
** RTREE library. All it does is declare the sqlite3RtreeInit() interface.
*/
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3RtreeInit(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
-500
View File
@@ -1,500 +0,0 @@
# 2008 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the r-tree extension.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source [file join [file dirname [info script]] rtree_util.tcl]
source $testdir/tester.tcl
# Test plan:
#
# rtree-1.*: Creating/destroying r-tree tables.
# rtree-2.*: Test the implicit constraints - unique rowid and
# (coord[N]<=coord[N+1]) for even values of N. Also
# automatic assigning of rowid values.
# rtree-3.*: Linear scans of r-tree data.
# rtree-4.*: Test INSERT
# rtree-5.*: Test DELETE
# rtree-6.*: Test UPDATE
# rtree-7.*: Test renaming an r-tree table.
# rtree-8.*: Test constrained scans of r-tree data.
#
# rtree-12.*: Test that on-conflict clauses are supported.
#
ifcapable !rtree {
finish_test
return
}
#----------------------------------------------------------------------------
# Test cases rtree-1.* test CREATE and DROP table statements.
#
# Test creating and dropping an rtree table.
#
do_test rtree-1.1.1 {
execsql { CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2) }
} {}
do_test rtree-1.1.2 {
execsql { SELECT name FROM sqlite_master ORDER BY name }
} {t1 t1_node t1_parent t1_rowid}
do_test rtree-1.1.3 {
execsql {
DROP TABLE t1;
SELECT name FROM sqlite_master ORDER BY name;
}
} {}
# Test creating and dropping an rtree table with an odd name in
# an attached database.
#
do_test rtree-1.2.1 {
file delete -force test2.db
execsql {
ATTACH 'test2.db' AS aux;
CREATE VIRTUAL TABLE aux.'a" "b' USING rtree(ii, x1, x2, y1, y2);
}
} {}
do_test rtree-1.2.2 {
execsql { SELECT name FROM sqlite_master ORDER BY name }
} {}
do_test rtree-1.2.3 {
execsql { SELECT name FROM aux.sqlite_master ORDER BY name }
} {{a" "b} {a" "b_node} {a" "b_parent} {a" "b_rowid}}
do_test rtree-1.2.4 {
execsql {
DROP TABLE aux.'a" "b';
SELECT name FROM aux.sqlite_master ORDER BY name;
}
} {}
# Test that the logic for checking the number of columns specified
# for an rtree table. Acceptable values are odd numbers between 3 and
# 11, inclusive.
#
set cols [list i1 i2 i3 i4 i5 i6 i7 i8 i9 iA iB iC iD iE iF iG iH iI iJ iK]
for {set nCol 1} {$nCol<[llength $cols]} {incr nCol} {
set columns [join [lrange $cols 0 [expr {$nCol-1}]] ,]
set X {0 {}}
if {$nCol%2 == 0} { set X {1 {Wrong number of columns for an rtree table}} }
if {$nCol < 3} { set X {1 {Too few columns for an rtree table}} }
if {$nCol > 11} { set X {1 {Too many columns for an rtree table}} }
do_test rtree-1.3.$nCol {
catchsql "
CREATE VIRTUAL TABLE t1 USING rtree($columns);
"
} $X
catchsql { DROP TABLE t1 }
}
# Test that it is possible to open an existing database that contains
# r-tree tables.
#
do_test rtree-1.4.1 {
execsql {
CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2);
INSERT INTO t1 VALUES(1, 5.0, 10.0);
INSERT INTO t1 VALUES(2, 15.0, 20.0);
}
} {}
do_test rtree-1.4.2 {
db close
sqlite3 db test.db
execsql { SELECT * FROM t1 ORDER BY ii }
} {1 5.0 10.0 2 15.0 20.0}
do_test rtree-1.4.3 {
execsql { DROP TABLE t1 }
} {}
# Test that it is possible to create an r-tree table with ridiculous
# column names.
#
do_test rtree-1.5.1 {
execsql {
CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim");
INSERT INTO t1 VALUES(1, 2, 3);
SELECT "the key", "x dim.", "x2'dim" FROM t1;
}
} {1 2.0 3.0}
do_test rtree-1.5.1 {
execsql { DROP TABLE t1 }
} {}
# Force the r-tree constructor to fail.
#
do_test rtree-1.6.1 {
execsql { CREATE TABLE t1_rowid(a); }
catchsql {
CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim");
}
} {1 {table "t1_rowid" already exists}}
do_test rtree-1.6.1 {
execsql { DROP TABLE t1_rowid }
} {}
#----------------------------------------------------------------------------
# Test cases rtree-2.*
#
do_test rtree-2.1.1 {
execsql {
CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2);
SELECT * FROM t1;
}
} {}
do_test rtree-2.1.2 {
execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
execsql { SELECT * FROM t1 }
} {1 1.0 3.0 2.0 4.0}
do_test rtree-2.1.3 {
execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
execsql { SELECT rowid FROM t1 ORDER BY rowid }
} {1 2}
do_test rtree-2.1.3 {
execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
execsql { SELECT ii FROM t1 ORDER BY ii }
} {1 2 3}
do_test rtree-2.2.1 {
catchsql { INSERT INTO t1 VALUES(2, 1, 3, 2, 4) }
} {1 {constraint failed}}
do_test rtree-2.2.2 {
catchsql { INSERT INTO t1 VALUES(4, 1, 3, 4, 2) }
} {1 {constraint failed}}
do_test rtree-2.2.3 {
catchsql { INSERT INTO t1 VALUES(4, 3, 1, 2, 4) }
} {1 {constraint failed}}
do_test rtree-2.2.4 {
execsql { SELECT ii FROM t1 ORDER BY ii }
} {1 2 3}
do_test rtree-2.X {
execsql { DROP TABLE t1 }
} {}
#----------------------------------------------------------------------------
# Test cases rtree-3.* test linear scans of r-tree table data. To test
# this we have to insert some data into an r-tree, but that is not the
# focus of these tests.
#
do_test rtree-3.1.1 {
execsql {
CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2);
SELECT * FROM t1;
}
} {}
do_test rtree-3.1.2 {
execsql {
INSERT INTO t1 VALUES(5, 1, 3, 2, 4);
SELECT * FROM t1;
}
} {5 1.0 3.0 2.0 4.0}
do_test rtree-3.1.3 {
execsql {
INSERT INTO t1 VALUES(6, 2, 6, 4, 8);
SELECT * FROM t1;
}
} {5 1.0 3.0 2.0 4.0 6 2.0 6.0 4.0 8.0}
# Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)):
do_test rtree-3.2.1 {
catchsql { INSERT INTO t1 VALUES(7, 2, 6, 4, 3) }
} {1 {constraint failed}}
do_test rtree-3.2.2 {
catchsql { INSERT INTO t1 VALUES(8, 2, 6, 3, 3) }
} {0 {}}
#----------------------------------------------------------------------------
# Test cases rtree-5.* test DELETE operations.
#
do_test rtree-5.1.1 {
execsql { CREATE VIRTUAL TABLE t2 USING rtree(ii, x1, x2) }
} {}
do_test rtree-5.1.2 {
execsql {
INSERT INTO t2 VALUES(1, 10, 20);
INSERT INTO t2 VALUES(2, 30, 40);
INSERT INTO t2 VALUES(3, 50, 60);
SELECT * FROM t2 ORDER BY ii;
}
} {1 10.0 20.0 2 30.0 40.0 3 50.0 60.0}
do_test rtree-5.1.3 {
execsql {
DELETE FROM t2 WHERE ii=2;
SELECT * FROM t2 ORDER BY ii;
}
} {1 10.0 20.0 3 50.0 60.0}
do_test rtree-5.1.4 {
execsql {
DELETE FROM t2 WHERE ii=1;
SELECT * FROM t2 ORDER BY ii;
}
} {3 50.0 60.0}
do_test rtree-5.1.5 {
execsql {
DELETE FROM t2 WHERE ii=3;
SELECT * FROM t2 ORDER BY ii;
}
} {}
do_test rtree-5.1.6 {
execsql { SELECT * FROM t2_rowid }
} {}
#----------------------------------------------------------------------------
# Test cases rtree-5.* test UPDATE operations.
#
do_test rtree-6.1.1 {
execsql { CREATE VIRTUAL TABLE t3 USING rtree(ii, x1, x2, y1, y2) }
} {}
do_test rtree-6.1.2 {
execsql {
INSERT INTO t3 VALUES(1, 2, 3, 4, 5);
UPDATE t3 SET x2=5;
SELECT * FROM t3;
}
} {1 2.0 5.0 4.0 5.0}
do_test rtree-6.1.3 {
execsql { UPDATE t3 SET ii = 2 }
execsql { SELECT * FROM t3 }
} {2 2.0 5.0 4.0 5.0}
#----------------------------------------------------------------------------
# Test cases rtree-7.* test rename operations.
#
do_test rtree-7.1.1 {
execsql {
CREATE VIRTUAL TABLE t4 USING rtree(ii, x1, x2, y1, y2, z1, z2);
INSERT INTO t4 VALUES(1, 2, 3, 4, 5, 6, 7);
}
} {}
do_test rtree-7.1.2 {
execsql { ALTER TABLE t4 RENAME TO t5 }
execsql { SELECT * FROM t5 }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.1.3 {
db close
sqlite3 db test.db
execsql { SELECT * FROM t5 }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.1.4 {
execsql { ALTER TABLE t5 RENAME TO 'raisara "one"'''}
execsql { SELECT * FROM "raisara ""one""'" }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.1.5 {
execsql { SELECT * FROM 'raisara "one"''' }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.1.6 {
execsql { ALTER TABLE "raisara ""one""'" RENAME TO "abc 123" }
execsql { SELECT * FROM "abc 123" }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.1.7 {
db close
sqlite3 db test.db
execsql { SELECT * FROM "abc 123" }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
# An error midway through a rename operation.
do_test rtree-7.2.1 {
execsql {
CREATE TABLE t4_node(a);
}
catchsql { ALTER TABLE "abc 123" RENAME TO t4 }
} {1 {SQL logic error or missing database}}
do_test rtree-7.2.2 {
execsql { SELECT * FROM "abc 123" }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.2.3 {
execsql {
DROP TABLE t4_node;
CREATE TABLE t4_rowid(a);
}
catchsql { ALTER TABLE "abc 123" RENAME TO t4 }
} {1 {SQL logic error or missing database}}
do_test rtree-7.2.4 {
db close
sqlite3 db test.db
execsql { SELECT * FROM "abc 123" }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
do_test rtree-7.2.5 {
execsql { DROP TABLE t4_rowid }
execsql { ALTER TABLE "abc 123" RENAME TO t4 }
execsql { SELECT * FROM t4 }
} {1 2.0 3.0 4.0 5.0 6.0 7.0}
#----------------------------------------------------------------------------
# Test cases rtree-8.*
#
# Test that the function to determine if a leaf cell is part of the
# result set works.
do_test rtree-8.1.1 {
execsql {
CREATE VIRTUAL TABLE t6 USING rtree(ii, x1, x2);
INSERT INTO t6 VALUES(1, 3, 7);
INSERT INTO t6 VALUES(2, 4, 6);
}
} {}
do_test rtree-8.1.2 { execsql { SELECT ii FROM t6 WHERE x1>2 } } {1 2}
do_test rtree-8.1.3 { execsql { SELECT ii FROM t6 WHERE x1>3 } } {2}
do_test rtree-8.1.4 { execsql { SELECT ii FROM t6 WHERE x1>4 } } {}
do_test rtree-8.1.5 { execsql { SELECT ii FROM t6 WHERE x1>5 } } {}
do_test rtree-8.1.6 { execsql { SELECT ii FROM t6 WHERE x1<3 } } {}
do_test rtree-8.1.7 { execsql { SELECT ii FROM t6 WHERE x1<4 } } {1}
do_test rtree-8.1.8 { execsql { SELECT ii FROM t6 WHERE x1<5 } } {1 2}
#----------------------------------------------------------------------------
# Test cases rtree-9.*
#
# Test that ticket #3549 is fixed.
do_test rtree-9.1 {
execsql {
CREATE TABLE foo (id INTEGER PRIMARY KEY);
CREATE VIRTUAL TABLE bar USING rtree (id, minX, maxX, minY, maxY);
INSERT INTO foo VALUES (null);
INSERT INTO foo SELECT null FROM foo;
INSERT INTO foo SELECT null FROM foo;
INSERT INTO foo SELECT null FROM foo;
INSERT INTO foo SELECT null FROM foo;
INSERT INTO foo SELECT null FROM foo;
INSERT INTO foo SELECT null FROM foo;
DELETE FROM foo WHERE id > 40;
INSERT INTO bar SELECT NULL, 0, 0, 0, 0 FROM foo;
}
} {}
# This used to crash.
do_test rtree-9.2 {
execsql {
SELECT count(*) FROM bar b1, bar b2, foo s1 WHERE s1.id = b1.id;
}
} {1600}
do_test rtree-9.3 {
execsql {
SELECT count(*) FROM bar b1, bar b2, foo s1
WHERE b1.minX <= b2.maxX AND s1.id = b1.id;
}
} {1600}
#-------------------------------------------------------------------------
# Ticket #3970: Check that the error message is meaningful when a
# keyword is used as a column name.
#
do_test rtree-10.1 {
catchsql { CREATE VIRTUAL TABLE t7 USING rtree(index, x1, y1, x2, y2) }
} {1 {near "index": syntax error}}
#-------------------------------------------------------------------------
# Test last_insert_rowid().
#
do_test rtree-11.1 {
execsql {
CREATE VIRTUAL TABLE t8 USING rtree(idx, x1, x2, y1, y2);
INSERT INTO t8 VALUES(1, 1.0, 1.0, 2.0, 2.0);
SELECT last_insert_rowid();
}
} {1}
do_test rtree-11.2 {
execsql {
INSERT INTO t8 VALUES(NULL, 1.0, 1.0, 2.0, 2.0);
SELECT last_insert_rowid();
}
} {2}
#-------------------------------------------------------------------------
# Test on-conflict clause handling.
#
db_delete_and_reopen
do_execsql_test 12.0 {
CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2);
INSERT INTO t1 VALUES(1, 1, 2, 3, 4);
INSERT INTO t1 VALUES(2, 2, 3, 4, 5);
INSERT INTO t1 VALUES(3, 3, 4, 5, 6);
CREATE TABLE source(idx, x1, x2, y1, y2);
INSERT INTO source VALUES(5, 8, 8, 8, 8);
INSERT INTO source VALUES(2, 7, 7, 7, 7);
}
db_save_and_close
foreach {tn sql_template testdata} {
1 "INSERT %CONF% INTO t1 VALUES(2, 7, 7, 7, 7)" {
ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
REPLACE 0 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7}
}
2 "INSERT %CONF% INTO t1 SELECT * FROM source" {
ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8}
FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8}
REPLACE 1 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8}
}
3 "UPDATE %CONF% t1 SET idx = 2 WHERE idx = 4" {
ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
REPLACE 1 0 {1 1 2 3 4 2 4 5 6 7 3 3 4 5 6}
}
3 "UPDATE %CONF% t1 SET idx = ((idx+1)%5)+1 WHERE idx > 2" {
ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6}
FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6}
REPLACE 1 0 {1 4 5 6 7 2 2 3 4 5 5 3 4 5 6}
}
4 "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" {
ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6}
ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
REPLACE 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7}
}
} {
foreach {mode uses error data} $testdata {
db_restore_and_reopen
set sql [string map [list %CONF% "OR $mode"] $sql_template]
set testname "12.$tn.[string tolower $mode]"
execsql {
BEGIN;
INSERT INTO t1 VALUES(4, 4, 5, 6, 7);
}
set res(0) {0 {}}
set res(1) {1 {constraint failed}}
do_catchsql_test $testname.1 $sql $res($error)
do_test $testname.2 [list sql_uses_stmt db $sql] $uses
do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data
do_test $testname.4 { rtree_check db t1 } 0
db close
}
}
finish_test
-150
View File
@@ -1,150 +0,0 @@
# 2008 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the r-tree extension.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source [file join [file dirname [info script]] rtree_util.tcl]
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
set ::NROW 1000
set ::NDEL 10
set ::NSELECT 100
if {[info exists G(isquick)] && $G(isquick)} {
set ::NROW 100
set ::NSELECT 10
}
foreach module {rtree_i32 rtree} {
for {set nDim 1} {$nDim <= 5} {incr nDim} {
do_test rtree2-$module.$nDim.1 {
set cols [list]
foreach c [list c0 c1 c2 c3 c4 c5 c6 c7 c8 c9] {
lappend cols "$c REAL"
}
set cols [join [lrange $cols 0 [expr {$nDim*2-1}]] ", "]
execsql "
CREATE VIRTUAL TABLE t1 USING ${module}(ii, $cols);
CREATE TABLE t2 (ii, $cols);
"
} {}
do_test rtree2-$module.$nDim.2 {
db transaction {
for {set ii 0} {$ii < $::NROW} {incr ii} {
#puts "Row $ii"
set values [list]
for {set jj 0} {$jj<$nDim*2} {incr jj} {
lappend values [expr int(rand()*1000)]
}
set values [join $values ,]
#puts [rtree_treedump db t1]
#puts "INSERT INTO t2 VALUES($ii, $values)"
set rc [catch {db eval "INSERT INTO t1 VALUES($ii, $values)"}]
if {$rc} {
incr ii -1
} else {
db eval "INSERT INTO t2 VALUES($ii, $values)"
}
#if {[rtree_check db t1]} {
#puts [rtree_treedump db t1]
#exit
#}
}
}
set t1 [execsql {SELECT * FROM t1 ORDER BY ii}]
set t2 [execsql {SELECT * FROM t2 ORDER BY ii}]
set rc [expr {$t1 eq $t2}]
if {$rc != 1} {
puts $t1
puts $t2
}
set rc
} {1}
do_test rtree2-$module.$nDim.3 {
rtree_check db t1
} 0
set OPS [list < > <= >= =]
for {set ii 0} {$ii < $::NSELECT} {incr ii} {
do_test rtree2-$module.$nDim.4.$ii.1 {
set where [list]
foreach look_three_dots! {. . .} {
set colidx [expr int(rand()*($nDim*2+1))-1]
if {$colidx<0} {
set col ii
} else {
set col "c$colidx"
}
set op [lindex $OPS [expr int(rand()*[llength $OPS])]]
set val [expr int(rand()*1000)]
lappend where "$col $op $val"
}
set where [join $where " AND "]
set t1 [execsql "SELECT * FROM t1 WHERE $where ORDER BY ii"]
set t2 [execsql "SELECT * FROM t2 WHERE $where ORDER BY ii"]
set rc [expr {$t1 eq $t2}]
if {$rc != 1} {
#puts $where
puts $t1
puts $t2
#puts [rtree_treedump db t1]
#breakpoint
#set t1 [execsql "SELECT * FROM t1 WHERE $where ORDER BY ii"]
#exit
}
set rc
} {1}
}
for {set ii 0} {$ii < $::NROW} {incr ii $::NDEL} {
#puts [rtree_treedump db t1]
do_test rtree2-$module.$nDim.5.$ii.1 {
execsql "DELETE FROM t2 WHERE ii <= $::ii"
execsql "DELETE FROM t1 WHERE ii <= $::ii"
set t1 [execsql {SELECT * FROM t1 ORDER BY ii}]
set t2 [execsql {SELECT * FROM t2 ORDER BY ii}]
set rc [expr {$t1 eq $t2}]
if {$rc != 1} {
puts $t1
puts $t2
}
set rc
} {1}
do_test rtree2-$module.$nDim.5.$ii.2 {
rtree_check db t1
} {0}
}
do_test rtree2-$module.$nDim.6 {
execsql {
DROP TABLE t1;
DROP TABLE t2;
}
} {}
}
}
finish_test
-237
View File
@@ -1,237 +0,0 @@
# 2008 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing that the r-tree correctly handles
# out-of-memory conditions.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
ifcapable !rtree {
finish_test
return
}
# Test summary:
#
# rtree3-1: Test OOM in simple CREATE TABLE, INSERT, DELETE and SELECT
# commands on an almost empty table.
#
# rtree3-2: Test OOM in a DROP TABLE command.
#
# rtree3-3a: Test OOM during a transaction to insert 100 pseudo-random rows.
#
# rtree3-3b: Test OOM during a transaction deleting all entries in the
# database constructed in [rtree3-3a] in pseudo-random order.
#
# rtree3-4a: OOM during "SELECT count(*) FROM ..." on a big table.
#
# rtree3-4b: OOM while deleting rows from a big table.
#
# rtree3-5: Test OOM while inserting rows into a big table.
#
# rtree3-6: Test OOM while deleting all rows of a table, one at a time.
#
# rtree3-7: OOM during an ALTER TABLE RENAME TABLE command.
#
# rtree3-8: Test OOM while registering the r-tree module with sqlite.
#
do_faultsim_test rtree3-1 -faults oom* -prep {
faultsim_delete_and_reopen
} -body {
execsql {
BEGIN TRANSACTION;
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
INSERT INTO rt VALUES(NULL, 13, 15, 17, 19);
DELETE FROM rt WHERE ii = 1;
SELECT * FROM rt;
SELECT ii FROM rt WHERE ii = 2;
COMMIT;
}
}
do_test rtree3-2.prep {
faultsim_delete_and_reopen
execsql {
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
}
faultsim_save_and_close
} {}
do_faultsim_test rtree3-2 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql { DROP TABLE rt }
}
do_malloc_test rtree3-3.prep {
faultsim_delete_and_reopen
execsql {
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
}
faultsim_save_and_close
} {}
do_faultsim_test rtree3-3a -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
db eval BEGIN
for {set ii 0} {$ii < 100} {incr ii} {
set f [expr rand()]
db eval {INSERT INTO rt VALUES(NULL, $f*10.0, $f*10.0, $f*15.0, $f*15.0)}
}
db eval COMMIT
}
faultsim_save_and_close
do_faultsim_test rtree3-3b -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
db eval BEGIN
for {set ii 0} {$ii < 100} {incr ii} {
set f [expr rand()]
db eval { DELETE FROM rt WHERE x1<($f*10.0) AND x1>($f*10.5) }
}
db eval COMMIT
}
do_test rtree3-4.prep {
faultsim_delete_and_reopen
execsql {
BEGIN;
PRAGMA page_size = 512;
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
}
for {set i 0} {$i < 1500} {incr i} {
execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
}
execsql { COMMIT }
faultsim_save_and_close
} {}
do_faultsim_test rtree3-4a -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
db eval { SELECT count(*) FROM rt }
} -test {
faultsim_test_result {0 1500}
}
do_faultsim_test rtree3-4b -faults oom-transient -prep {
faultsim_restore_and_reopen
} -body {
db eval { DELETE FROM rt WHERE ii BETWEEN 1 AND 100 }
} -test {
faultsim_test_result {0 {}}
}
do_test rtree3-5.prep {
faultsim_delete_and_reopen
execsql {
BEGIN;
PRAGMA page_size = 512;
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
}
for {set i 0} {$i < 100} {incr i} {
execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
}
execsql { COMMIT }
faultsim_save_and_close
} {}
do_faultsim_test rtree3-5 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
for {set i 100} {$i < 110} {incr i} {
execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
}
} -test {
faultsim_test_result {0 {}}
}
do_test rtree3-6.prep {
faultsim_delete_and_reopen
execsql {
BEGIN;
PRAGMA page_size = 512;
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
}
for {set i 0} {$i < 50} {incr i} {
execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
}
execsql { COMMIT }
faultsim_save_and_close
} {}
do_faultsim_test rtree3-6 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql BEGIN
for {set i 0} {$i < 50} {incr i} {
execsql { DELETE FROM rt WHERE ii=$i }
}
execsql COMMIT
} -test {
faultsim_test_result {0 {}}
}
do_test rtree3-7.prep {
faultsim_delete_and_reopen
execsql { CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2) }
faultsim_save_and_close
} {}
do_faultsim_test rtree3-7 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { ALTER TABLE rt RENAME TO rt2 }
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test rtree3-8 -faults oom-* -prep {
catch { db close }
} -body {
sqlite3 db test.db
}
do_faultsim_test rtree3-9 -faults oom-* -prep {
sqlite3 db :memory:
} -body {
set rc [register_cube_geom db]
if {$rc != "SQLITE_OK"} { error $rc }
} -test {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}
do_test rtree3-10.prep {
faultsim_delete_and_reopen
execsql {
CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2, z1, z2);
INSERT INTO rt VALUES(1, 10, 10, 10, 11, 11, 11);
INSERT INTO rt VALUES(2, 5, 6, 6, 7, 7, 8);
}
faultsim_save_and_close
} {}
do_faultsim_test rtree3-10 -faults oom-* -prep {
faultsim_restore_and_reopen
register_cube_geom db
execsql { SELECT * FROM rt }
} -body {
execsql { SELECT ii FROM rt WHERE ii MATCH cube(4.5, 5.5, 6.5, 1, 1, 1) }
} -test {
faultsim_test_result {0 2}
}
finish_test
-234
View File
@@ -1,234 +0,0 @@
# 2008 May 23
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Randomized test cases for the rtree extension.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
set ::NROW 2500
if {[info exists G(isquick)] && $G(isquick)} {
set ::NROW 250
}
# Return a floating point number between -X and X.
#
proc rand {X} {
return [expr {int((rand()-0.5)*1024.0*$X)/512.0}]
}
# Return a positive floating point number less than or equal to X
#
proc randincr {X} {
while 1 {
set r [expr {int(rand()*$X*32.0)/32.0}]
if {$r>0.0} {return $r}
}
}
# Scramble the $inlist into a random order.
#
proc scramble {inlist} {
set y {}
foreach x $inlist {
lappend y [list [expr {rand()}] $x]
}
set y [lsort $y]
set outlist {}
foreach x $y {
lappend outlist [lindex $x 1]
}
return $outlist
}
# Always use the same random seed so that the sequence of tests
# is repeatable.
#
expr {srand(1234)}
# Run these tests for all number of dimensions between 1 and 5.
#
for {set nDim 1} {$nDim<=5} {incr nDim} {
# Construct an rtree virtual table and an ordinary btree table
# to mirror it. The ordinary table should be much slower (since
# it has to do a full table scan) but should give the exact same
# answers.
#
do_test rtree4-$nDim.1 {
set clist {}
set cklist {}
for {set i 0} {$i<$nDim} {incr i} {
lappend clist mn$i mx$i
lappend cklist "mn$i<mx$i"
}
db eval "DROP TABLE IF EXISTS rx"
db eval "DROP TABLE IF EXISTS bx"
db eval "CREATE VIRTUAL TABLE rx USING rtree(id, [join $clist ,])"
db eval "CREATE TABLE bx(id INTEGER PRIMARY KEY,\
[join $clist ,], CHECK( [join $cklist { AND }] ))"
} {}
# Do many insertions of small objects. Do both overlapping and
# contained-within queries after each insert to verify that all
# is well.
#
unset -nocomplain where
for {set i 1} {$i<$::NROW} {incr i} {
# Do a random insert
#
do_test rtree4-$nDim.2.$i.1 {
set vlist {}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 50]}]
lappend vlist $mn $mx
}
db eval "INSERT INTO rx VALUES(NULL, [join $vlist ,])"
db eval "INSERT INTO bx VALUES(NULL, [join $vlist ,])"
} {}
# Do a contained-in query on all dimensions
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mn$j>=$mn mx$j<=$mx
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.2 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do an overlaps query on all dimensions
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mx$j>=$mn mn$j<=$mx
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.3 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do a contained-in query with surplus contraints at the beginning.
# This should force a full-table scan on the rtree.
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
lappend where mn$j>-10000 mx$j<10000
}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mn$j>=$mn mx$j<=$mx
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.3 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do an overlaps query with surplus contraints at the beginning.
# This should force a full-table scan on the rtree.
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
lappend where mn$j>=-10000 mx$j<=10000
}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mx$j>$mn mn$j<$mx
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.4 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do a contained-in query with surplus contraints at the end
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mn$j>=$mn mx$j<$mx
}
for {set j [expr {$nDim-1}]} {$j>=0} {incr j -1} {
lappend where mn$j>=-10000 mx$j<10000
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.5 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do an overlaps query with surplus contraints at the end
#
set where {}
for {set j [expr {$nDim-1}]} {$j>=0} {incr j -1} {
set mn [rand 10000]
set mx [expr {$mn+[randincr 500]}]
lappend where mx$j>$mn mn$j<=$mx
}
for {set j 0} {$j<$nDim} {incr j} {
lappend where mx$j>-10000 mn$j<=10000
}
set where "WHERE [join $where { AND }]"
do_test rtree4-$nDim.2.$i.6 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do a contained-in query with surplus contraints where the
# constraints appear in a random order.
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
set mn1 [rand 10000]
set mn2 [expr {$mn1+[randincr 100]}]
set mx1 [expr {$mn2+[randincr 400]}]
set mx2 [expr {$mx1+[randincr 100]}]
lappend where mn$j>=$mn1 mn$j>$mn2 mx$j<$mx1 mx$j<=$mx2
}
set where "WHERE [join [scramble $where] { AND }]"
do_test rtree4-$nDim.2.$i.7 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
# Do an overlaps query with surplus contraints where the
# constraints appear in a random order.
#
set where {}
for {set j 0} {$j<$nDim} {incr j} {
set mn1 [rand 10000]
set mn2 [expr {$mn1+[randincr 100]}]
set mx1 [expr {$mn2+[randincr 400]}]
set mx2 [expr {$mx1+[randincr 100]}]
lappend where mx$j>=$mn1 mx$j>$mn2 mn$j<$mx1 mn$j<=$mx2
}
set where "WHERE [join [scramble $where] { AND }]"
do_test rtree4-$nDim.2.$i.8 {
list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
} [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
}
}
finish_test
-78
View File
@@ -1,78 +0,0 @@
# 2008 Jul 14
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the r-tree extension when it is
# configured to store values as 32 bit integers.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
do_test rtree5-1.0 {
execsql { CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2, y1, y2) }
} {}
do_test rtree5-1.1 {
execsql { INSERT INTO t1 VALUES(1, 5, 10, 4, 11.2) }
} {}
do_test rtree5-1.2 {
execsql { SELECT * FROM t1 }
} {1 5 10 4 11}
do_test rtree5-1.3 {
execsql { SELECT typeof(x1) FROM t1 }
} {integer}
do_test rtree5-1.4 {
execsql { SELECT x1==5 FROM t1 }
} {1}
do_test rtree5-1.5 {
execsql { SELECT x1==5.2 FROM t1 }
} {0}
do_test rtree5-1.6 {
execsql { SELECT x1==5.0 FROM t1 }
} {1}
do_test rtree5-1.7 {
execsql { SELECT count(*) FROM t1 WHERE x1==5 }
} {1}
do_test rtree5-1.8 {
execsql { SELECT count(*) FROM t1 WHERE x1==5.2 }
} {0}
do_test rtree5-1.9 {
execsql { SELECT count(*) FROM t1 WHERE x1==5.0 }
} {1}
do_test rtree5-1.10 {
execsql { SELECT (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5 }
} {2147483643 2147483647 -2147483648 -2147483643}
do_test rtree5-1.10 {
execsql {
INSERT INTO t1 VALUES(2, (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5)
}
} {}
do_test rtree5-1.12 {
execsql { SELECT * FROM t1 WHERE id=2 }
} {2 2147483643 2147483647 -2147483648 -2147483643}
do_test rtree5-1.13 {
execsql {
SELECT * FROM t1 WHERE
x1=2147483643 AND x2=2147483647 AND
y1=-2147483648 AND y2=-2147483643
}
} {2 2147483643 2147483647 -2147483648 -2147483643}
finish_test
-156
View File
@@ -1,156 +0,0 @@
# 2008 Sep 1
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
# Operator Byte Value
# ----------------------
# = 0x41 ('A')
# <= 0x42 ('B')
# < 0x43 ('C')
# >= 0x44 ('D')
# > 0x45 ('E')
# ----------------------
proc rtree_strategy {sql} {
set ret [list]
db eval "explain $sql" a {
if {$a(opcode) eq "VFilter"} {
lappend ret $a(p4)
}
}
set ret
}
proc query_plan {sql} {
set ret [list]
db eval "explain query plan $sql" a {
lappend ret $a(detail)
}
set ret
}
do_test rtree6-1.1 {
execsql {
CREATE TABLE t2(k INTEGER PRIMARY KEY, v);
CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2);
}
} {}
do_test rtree6-1.2 {
rtree_strategy {SELECT * FROM t1 WHERE x1>10}
} {Ea}
do_test rtree6-1.3 {
rtree_strategy {SELECT * FROM t1 WHERE x1<10}
} {Ca}
do_test rtree6-1.4 {
rtree_strategy {SELECT * FROM t1,t2 WHERE k=ii AND x1<10}
} {Ca}
do_test rtree6-1.5 {
rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10}
} {Ca}
do_eqp_test rtree6.2.1 {
SELECT * FROM t1,t2 WHERE k=+ii AND x1<10
} {
0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)}
0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
}
do_eqp_test rtree6.2.2 {
SELECT * FROM t1,t2 WHERE k=ii AND x1<10
} {
0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)}
0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
}
do_eqp_test rtree6.2.3 {
SELECT * FROM t1,t2 WHERE k=ii
} {
0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)}
0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
}
do_eqp_test rtree6.2.4 {
SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
} {
0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~0 rows)}
0 1 1 {SCAN TABLE t2 (~100000 rows)}
}
do_eqp_test rtree6.2.5 {
SELECT * FROM t1,t2 WHERE k=ii AND x1<v
} {
0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)}
0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
}
do_execsql_test rtree6-3.1 {
CREATE VIRTUAL TABLE t3 USING rtree(id, x1, x2, y1, y2);
INSERT INTO t3 VALUES(NULL, 1, 1, 2, 2);
SELECT * FROM t3 WHERE
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5;
} {1 1.0 1.0 2.0 2.0}
do_test rtree6.3.2 {
rtree_strategy {
SELECT * FROM t3 WHERE
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5
}
} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa}
do_test rtree6.3.3 {
rtree_strategy {
SELECT * FROM t3 WHERE
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5
}
} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa}
do_execsql_test rtree6-3.4 {
SELECT * FROM t3 WHERE x1>0.5 AND x1>0.8 AND x1>1.1
} {}
do_execsql_test rtree6-3.5 {
SELECT * FROM t3 WHERE
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND
x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>1.1
} {}
finish_test
-58
View File
@@ -1,58 +0,0 @@
# 2010 February 16
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Test that nothing goes wrong if an rtree table is created, then the
# database page-size is modified. At one point (3.6.22), this was causing
# malfunctions.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree||!vacuum {
finish_test
return
}
do_test rtree7-1.1 {
execsql {
PRAGMA page_size = 1024;
CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2, y1, y2);
INSERT INTO rt VALUES(1, 1, 2, 3, 4);
}
} {}
do_test rtree7-1.2 {
execsql { SELECT * FROM rt }
} {1 1.0 2.0 3.0 4.0}
do_test rtree7-1.3 {
execsql {
PRAGMA page_size = 2048;
VACUUM;
SELECT * FROM rt;
}
} {1 1.0 2.0 3.0 4.0}
do_test rtree7-1.4 {
for {set i 2} {$i <= 51} {incr i} {
execsql { INSERT INTO rt VALUES($i, 1, 2, 3, 4) }
}
execsql { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt }
} {51.0 102.0 153.0 204.0}
do_test rtree7-1.5 {
execsql {
PRAGMA page_size = 512;
VACUUM;
SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt
}
} {51.0 102.0 153.0 204.0}
finish_test
-171
View File
@@ -1,171 +0,0 @@
# 2010 February 16
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
#-------------------------------------------------------------------------
# The following block of tests - rtree8-1.* - feature reading and writing
# an r-tree table while there exist open cursors on it.
#
proc populate_t1 {n} {
execsql { DELETE FROM t1 }
for {set i 1} {$i <= $n} {incr i} {
execsql { INSERT INTO t1 VALUES($i, $i, $i+2) }
}
}
# A DELETE while a cursor is reading the table.
#
do_test rtree8-1.1.1 {
execsql { PRAGMA page_size = 512 }
execsql { CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2) }
populate_t1 5
} {}
do_test rtree8-1.1.2 {
set res [list]
db eval { SELECT * FROM t1 } {
lappend res $x1 $x2
if {$id==3} { db eval { DELETE FROM t1 WHERE id>3 } }
}
set res
} {1 3 2 4 3 5}
do_test rtree8-1.1.3 {
execsql { SELECT * FROM t1 }
} {1 1 3 2 2 4 3 3 5}
# Many SELECTs on the same small table.
#
proc nested_select {n} {
set ::max $n
db eval { SELECT * FROM t1 } {
if {$id == $n} { nested_select [expr $n+1] }
}
return $::max
}
do_test rtree8-1.2.1 { populate_t1 50 } {}
do_test rtree8-1.2.2 { nested_select 1 } {51}
# This test runs many SELECT queries simultaneously against a large
# table, causing a collision in the hash-table used to store r-tree
# nodes internally.
#
populate_t1 1500
do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {164}
do_test rtree8-1.3.2 {
set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}]
set stmt_list [list]
foreach row $rowids {
set stmt [sqlite3_prepare db "SELECT * FROM t1 WHERE id = $row" -1 tail]
sqlite3_step $stmt
lappend res_list [sqlite3_column_int $stmt 0]
lappend stmt_list $stmt
}
} {}
do_test rtree8-1.3.3 { set res_list } $rowids
do_execsql_test rtree8-1.3.4 { SELECT count(*) FROM t1 } {1500}
do_test rtree8-1.3.5 {
foreach stmt $stmt_list { sqlite3_finalize $stmt }
} {}
#-------------------------------------------------------------------------
# The following block of tests - rtree8-2.* - test a couple of database
# corruption cases. In this case things are not corrupted at the b-tree
# level, but the contents of the various tables used internally by an
# r-tree table are inconsistent.
#
populate_t1 50
do_execsql_test rtree8-2.1.1 { SELECT max(nodeno) FROM t1_node } {5}
do_execsql_test rtree8-2.1.2 { DELETE FROM t1_node } {}
for {set i 1} {$i <= 50} {incr i} {
do_catchsql_test rtree8-2.1.3.$i {
SELECT * FROM t1 WHERE id = $i
} {1 {database disk image is malformed}}
}
do_catchsql_test rtree8-2.1.4 {
SELECT * FROM t1
} {1 {database disk image is malformed}}
do_catchsql_test rtree8-2.1.5 {
DELETE FROM t1
} {1 {database disk image is malformed}}
do_execsql_test rtree8-2.1.6 {
DROP TABLE t1;
CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
} {}
populate_t1 50
do_execsql_test rtree8-2.2.1 {
DELETE FROM t1_parent
} {}
do_catchsql_test rtree8-2.2.2 {
DELETE FROM t1 WHERE id=25
} {1 {database disk image is malformed}}
do_execsql_test rtree8-2.2.3 {
DROP TABLE t1;
CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
} {}
#-------------------------------------------------------------------------
# Test that trying to use the MATCH operator with the r-tree module does
# not confuse it.
#
populate_t1 10
do_catchsql_test rtree8-3.1 {
SELECT * FROM t1 WHERE x1 MATCH '1234'
} {1 {SQL logic error or missing database}}
#-------------------------------------------------------------------------
# Test a couple of invalid arguments to rtreedepth().
#
do_catchsql_test rtree8-4.1 {
SELECT rtreedepth('hello world')
} {1 {Invalid argument to rtreedepth()}}
do_catchsql_test rtree8-4.2 {
SELECT rtreedepth(X'00')
} {1 {Invalid argument to rtreedepth()}}
#-------------------------------------------------------------------------
# Delete half of a lopsided tree.
#
do_execsql_test rtree8-5.1 {
CREATE VIRTUAL TABLE t2 USING rtree_i32(id, x1, x2)
} {}
do_test rtree8-5.2 {
execsql BEGIN
for {set i 0} {$i < 100} {incr i} {
execsql { INSERT INTO t2 VALUES($i, 100, 101) }
}
for {set i 100} {$i < 200} {incr i} {
execsql { INSERT INTO t2 VALUES($i, 1000, 1001) }
}
execsql COMMIT
} {}
do_test rtree8-5.3 {
execsql BEGIN
for {set i 0} {$i < 200} {incr i} {
execsql { DELETE FROM t2 WHERE id = $i }
}
execsql COMMIT
} {}
finish_test
-125
View File
@@ -1,125 +0,0 @@
# 2010 August 28
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file contains tests for the r-tree module. Specifically, it tests
# that custom r-tree queries (geometry callbacks) work.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
register_cube_geom db
do_execsql_test rtree9-1.1 {
CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2, y1, y2, z1, z2);
INSERT INTO rt VALUES(1, 1, 2, 1, 2, 1, 2);
} {}
do_execsql_test rtree9-1.2 {
SELECT * FROM rt WHERE id MATCH cube(0, 0, 0, 2, 2, 2);
} {1 1.0 2.0 1.0 2.0 1.0 2.0}
do_execsql_test rtree9-1.3 {
SELECT * FROM rt WHERE id MATCH cube(3, 3, 3, 2, 2, 2);
} {}
do_execsql_test rtree9-1.4 {
DELETE FROM rt;
} {}
for {set i 0} {$i < 1000} {incr i} {
set x [expr $i%10]
set y [expr ($i/10)%10]
set z [expr ($i/100)%10]
execsql { INSERT INTO rt VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
}
do_execsql_test rtree9-2.1 {
SELECT id FROM rt WHERE id MATCH cube(2.5, 2.5, 2.5, 1, 1, 1) ORDER BY id;
} {222 223 232 233 322 323 332 333}
do_execsql_test rtree9-2.2 {
SELECT id FROM rt WHERE id MATCH cube(5.5, 5.5, 5.5, 1, 1, 1) ORDER BY id;
} {555 556 565 566 655 656 665 666}
do_execsql_test rtree9-3.1 {
CREATE VIRTUAL TABLE rt32 USING rtree_i32(id, x1, x2, y1, y2, z1, z2);
} {}
for {set i 0} {$i < 1000} {incr i} {
set x [expr $i%10]
set y [expr ($i/10)%10]
set z [expr ($i/100)%10]
execsql { INSERT INTO rt32 VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
}
do_execsql_test rtree9-3.2 {
SELECT id FROM rt32 WHERE id MATCH cube(3, 3, 3, 1, 1, 1) ORDER BY id;
} {222 223 224 232 233 234 242 243 244 322 323 324 332 333 334 342 343 344 422 423 424 432 433 434 442 443 444}
do_execsql_test rtree9-3.3 {
SELECT id FROM rt32 WHERE id MATCH cube(5.5, 5.5, 5.5, 1, 1, 1) ORDER BY id;
} {555 556 565 566 655 656 665 666}
do_catchsql_test rtree9-4.1 {
SELECT id FROM rt32 WHERE id MATCH cube(5.5, 5.5, 1, 1, 1) ORDER BY id;
} {1 {SQL logic error or missing database}}
for {set x 2} {$x<200} {incr x 2} {
do_catchsql_test rtree9-4.2.[expr $x/2] {
SELECT id FROM rt WHERE id MATCH randomblob($x)
} {1 {SQL logic error or missing database}}
}
do_catchsql_test rtree9-4.3 {
SELECT id FROM rt WHERE id MATCH CAST(
(cube(5.5, 5.5, 5.5, 1, 1, 1) || X'1234567812345678') AS blob
)
} {1 {SQL logic error or missing database}}
#-------------------------------------------------------------------------
# Test the example 2d "circle" geometry callback.
#
register_circle_geom db
breakpoint
do_execsql_test rtree9-5.1 {
CREATE VIRTUAL TABLE rt2 USING rtree(id, xmin, xmax, ymin, ymax);
INSERT INTO rt2 VALUES(1, 1, 2, 1, 2);
INSERT INTO rt2 VALUES(2, 1, 2, -2, -1);
INSERT INTO rt2 VALUES(3, -2, -1, -2, -1);
INSERT INTO rt2 VALUES(4, -2, -1, 1, 2);
INSERT INTO rt2 VALUES(5, 2, 3, 2, 3);
INSERT INTO rt2 VALUES(6, 2, 3, -3, -2);
INSERT INTO rt2 VALUES(7, -3, -2, -3, -2);
INSERT INTO rt2 VALUES(8, -3, -2, 2, 3);
INSERT INTO rt2 VALUES(9, 1.8, 3, 1.8, 3);
INSERT INTO rt2 VALUES(10, 1.8, 3, -3, -1.8);
INSERT INTO rt2 VALUES(11, -3, -1.8, -3, -1.8);
INSERT INTO rt2 VALUES(12, -3, -1.8, 1.8, 3);
INSERT INTO rt2 VALUES(13, -15, 15, 1.8, 2.2);
INSERT INTO rt2 VALUES(14, -15, 15, -2.2, -1.8);
INSERT INTO rt2 VALUES(15, 1.8, 2.2, -15, 15);
INSERT INTO rt2 VALUES(16, -2.2, -1.8, -15, 15);
INSERT INTO rt2 VALUES(17, -100, 100, -100, 100);
} {}
do_execsql_test rtree9-5.2 {
SELECT id FROM rt2 WHERE id MATCH circle(0.0, 0.0, 2.0);
} {1 2 3 4 13 14 15 16 17}
do_execsql_test rtree9-5.3 {
UPDATE rt2 SET xmin=xmin+5, ymin=ymin+5, xmax=xmax+5, ymax=ymax+5;
SELECT id FROM rt2 WHERE id MATCH circle(5.0, 5.0, 2.0);
} {1 2 3 4 13 14 15 16 17}
finish_test
-220
View File
@@ -1,220 +0,0 @@
# 2010 September 22
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file contains tests for the r-tree module. Specifically, it tests
# that corrupt or inconsistent databases do not cause crashes in the r-tree
# module.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
proc create_t1 {} {
db close
forcedelete test.db
sqlite3 db test.db
execsql {
PRAGMA page_size = 1024;
CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2, y1, y2);
}
}
proc populate_t1 {} {
execsql BEGIN
for {set i 0} {$i < 500} {incr i} {
set x2 [expr $i+5]
set y2 [expr $i+5]
execsql { INSERT INTO t1 VALUES($i, $i, $x2, $i, $y2) }
}
execsql COMMIT
}
proc truncate_node {nodeno nTrunc} {
set blob [db one {SELECT data FROM t1_node WHERE nodeno=$nodeno}]
if {$nTrunc<0} {set nTrunc "end-$nTrunc"}
set blob [string range $blob 0 $nTrunc]
db eval { UPDATE t1_node SET data = $blob WHERE nodeno=$nodeno }
}
proc set_tree_depth {tbl {newvalue ""}} {
set blob [db one "SELECT data FROM ${tbl}_node WHERE nodeno=1"]
if {$newvalue == ""} {
binary scan $blob Su oldvalue
return $oldvalue
}
set blob [binary format Sua* $newvalue [string range $blob 2 end]]
db eval "UPDATE ${tbl}_node SET data = \$blob WHERE nodeno=1"
return [set_tree_depth $tbl]
}
proc set_entry_count {tbl nodeno {newvalue ""}} {
set blob [db one "SELECT data FROM ${tbl}_node WHERE nodeno=$nodeno"]
if {$newvalue == ""} {
binary scan [string range $blob 2 end] Su oldvalue
return $oldvalue
}
set blob [binary format a*Sua* \
[string range $blob 0 1] $newvalue [string range $blob 4 end]
]
db eval "UPDATE ${tbl}_node SET data = \$blob WHERE nodeno=$nodeno"
return [set_entry_count $tbl $nodeno]
}
proc do_corruption_tests {prefix args} {
set testarray [lindex $args end]
set errormsg {database disk image is malformed}
foreach {z value} [lrange $args 0 end-1] {
set n [string length $z]
if {$n>=2 && [string equal -length $n $z "-error"]} {
set errormsg $value
}
}
foreach {tn sql} $testarray {
do_catchsql_test $prefix.$tn $sql [list 1 $errormsg]
}
}
#-------------------------------------------------------------------------
# Test the libraries response if the %_node table is completely empty
# (i.e. the root node is missing), or has been removed from the database
# entirely.
#
create_t1
populate_t1
do_execsql_test rtreeA-1.0 {
DELETE FROM t1_node;
} {}
do_corruption_tests rtreeA-1.1 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
4 "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}
do_execsql_test rtreeA-1.2.0 { DROP TABLE t1_node } {}
do_corruption_tests rtreeA-1.2 -error "SQL logic error or missing database" {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
4 "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}
#-------------------------------------------------------------------------
# Test the libraries response if some of the entries in the %_node table
# are the wrong size.
#
create_t1
populate_t1
do_test rtreeA-2.1.0 {
set nodes [db eval {select nodeno FROM t1_node}]
foreach {a b c} $nodes { truncate_node $c 200 }
} {}
do_corruption_tests rtreeA-2.1 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
4 "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}
create_t1
populate_t1
do_test rtreeA-2.2.0 { truncate_node 1 200 } {}
do_corruption_tests rtreeA-2.2 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
4 "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}
#-------------------------------------------------------------------------
# Set the "depth" of the tree stored on the root node incorrectly. Test
# that this does not cause any problems.
#
create_t1
populate_t1
do_test rtreeA-3.1.0.1 { set_tree_depth t1 } {1}
do_test rtreeA-3.1.0.2 { set_tree_depth t1 3 } {3}
do_corruption_tests rtreeA-3.1 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
}
do_test rtreeA-3.2.0 { set_tree_depth t1 1000 } {1000}
do_corruption_tests rtreeA-3.2 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
}
create_t1
populate_t1
do_test rtreeA-3.3.0 {
execsql { DELETE FROM t1 WHERE rowid = 0 }
set_tree_depth t1 65535
} {65535}
do_corruption_tests rtreeA-3.3 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
}
#-------------------------------------------------------------------------
# Set the "number of entries" field on some nodes incorrectly.
#
create_t1
populate_t1
do_test rtreeA-4.1.0 {
set_entry_count t1 1 4000
} {4000}
do_corruption_tests rtreeA-4.1 {
1 "SELECT * FROM t1"
2 "SELECT * FROM t1 WHERE rowid=5"
3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
4 "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}
#-------------------------------------------------------------------------
# Remove entries from the %_parent table and check that this does not
# cause a crash.
#
create_t1
populate_t1
do_execsql_test rtreeA-5.1.0 { DELETE FROM t1_parent } {}
do_corruption_tests rtreeA-5.1 {
1 "DELETE FROM t1 WHERE rowid = 5"
2 "DELETE FROM t1"
}
#-------------------------------------------------------------------------
# Add some bad entries to the %_parent table.
#
create_t1
populate_t1
do_execsql_test rtreeA-6.1.0 {
UPDATE t1_parent set parentnode = parentnode+1
} {}
do_corruption_tests rtreeA-6.1 {
1 "DELETE FROM t1 WHERE rowid = 5"
2 "UPDATE t1 SET x1=x1+1, x2=x2+1"
}
finish_test
-34
View File
@@ -1,34 +0,0 @@
# 2011 March 2
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# Make sure the rtreenode() testing function can handle entries with
# 64-bit rowids.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
do_test rtreeB-1.1 {
db eval {
CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1);
INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0);
INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0);
INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0);
INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0);
INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400);
SELECT rtreenode(2, data) FROM t1_node;
}
} {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}}
finish_test
-74
View File
@@ -1,74 +0,0 @@
set testdir [file join [file dirname $argv0] .. .. test]
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
set NROW 10000
set NQUERY 500
puts "Generating $NROW rows of data..."
set data [list]
for {set ii 0} {$ii < $NROW} {incr ii} {
set x1 [expr {rand()*1000}]
set x2 [expr {$x1+rand()*50}]
set y1 [expr {rand()*1000}]
set y2 [expr {$y1+rand()*50}]
lappend data $x1 $x2 $y1 $y2
}
puts "Finished generating data"
set sql1 {CREATE TABLE btree(ii INTEGER PRIMARY KEY, x1, x2, y1, y2)}
set sql2 {CREATE VIRTUAL TABLE rtree USING rtree(ii, x1, x2, y1, y2)}
puts "Creating tables:"
puts " $sql1"
puts " $sql2"
db eval $sql1
db eval $sql2
db eval "pragma cache_size=100"
puts -nonewline "Inserting into btree... "
flush stdout
set btree_time [time {db transaction {
set ii 1
foreach {x1 x2 y1 y2} $data {
db eval {INSERT INTO btree VALUES($ii, $x1, $x2, $y1, $y2)}
incr ii
}
}}]
puts "$btree_time"
puts -nonewline "Inserting into rtree... "
flush stdout
set rtree_time [time {db transaction {
set ii 1
foreach {x1 x2 y1 y2} $data {
incr ii
db eval {INSERT INTO rtree VALUES($ii, $x1, $x2, $y1, $y2)}
}
}}]
puts "$rtree_time"
puts -nonewline "Selecting from btree... "
flush stdout
set btree_select_time [time {
foreach {x1 x2 y1 y2} [lrange $data 0 [expr $NQUERY*4-1]] {
db eval {SELECT * FROM btree WHERE x1<$x1 AND x2>$x2 AND y1<$y1 AND y2>$y2}
}
}]
puts "$btree_select_time"
puts -nonewline "Selecting from rtree... "
flush stdout
set rtree_select_time [time {
foreach {x1 x2 y1 y2} [lrange $data 0 [expr $NQUERY*4-1]] {
db eval {SELECT * FROM rtree WHERE x1<$x1 AND x2>$x2 AND y1<$y1 AND y2>$y2}
}
}]
puts "$rtree_select_time"
-192
View File
@@ -1,192 +0,0 @@
# 2008 Feb 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains Tcl code that may be useful for testing or
# analyzing r-tree structures created with this module. It is
# used by both test procedures and the r-tree viewer application.
#
#--------------------------------------------------------------------------
# PUBLIC API:
#
# rtree_depth
# rtree_ndim
# rtree_node
# rtree_mincells
# rtree_check
# rtree_dump
# rtree_treedump
#
proc rtree_depth {db zTab} {
$db one "SELECT rtreedepth(data) FROM ${zTab}_node WHERE nodeno=1"
}
proc rtree_nodedepth {db zTab iNode} {
set iDepth [rtree_depth $db $zTab]
set ii $iNode
while {$ii != 1} {
set sql "SELECT parentnode FROM ${zTab}_parent WHERE nodeno = $ii"
set ii [db one $sql]
incr iDepth -1
}
return $iDepth
}
# Return the number of dimensions of the rtree.
#
proc rtree_ndim {db zTab} {
set nDim [expr {(([llength [$db eval "pragma table_info($zTab)"]]/6)-1)/2}]
}
# Return the contents of rtree node $iNode.
#
proc rtree_node {db zTab iNode {iPrec 6}} {
set nDim [rtree_ndim $db $zTab]
set sql "
SELECT rtreenode($nDim, data) FROM ${zTab}_node WHERE nodeno = $iNode
"
set node [db one $sql]
set nCell [llength $node]
set nCoord [expr $nDim*2]
for {set ii 0} {$ii < $nCell} {incr ii} {
for {set jj 1} {$jj <= $nCoord} {incr jj} {
set newval [format "%.${iPrec}f" [lindex $node $ii $jj]]
lset node $ii $jj $newval
}
}
set node
}
proc rtree_mincells {db zTab} {
set n [$db one "select length(data) FROM ${zTab}_node LIMIT 1"]
set nMax [expr {int(($n-4)/(8+[rtree_ndim $db $zTab]*2*4))}]
return [expr {int($nMax/3)}]
}
# An integrity check for the rtree $zTab accessible via database
# connection $db.
#
proc rtree_check {db zTab} {
array unset ::checked
# Check each r-tree node.
set rc [catch {
rtree_node_check $db $zTab 1 [rtree_depth $db $zTab]
} msg]
if {$rc && $msg ne ""} { error $msg }
# Check that the _rowid and _parent tables have the right
# number of entries.
set nNode [$db one "SELECT count(*) FROM ${zTab}_node"]
set nRow [$db one "SELECT count(*) FROM ${zTab}"]
set nRowid [$db one "SELECT count(*) FROM ${zTab}_rowid"]
set nParent [$db one "SELECT count(*) FROM ${zTab}_parent"]
if {$nNode != ($nParent+1)} {
error "Wrong number of entries in ${zTab}_parent"
}
if {$nRow != $nRowid} {
error "Wrong number of entries in ${zTab}_rowid"
}
return $rc
}
proc rtree_node_check {db zTab iNode iDepth} {
if {[info exists ::checked($iNode)]} { error "Second ref to $iNode" }
set ::checked($iNode) 1
set node [rtree_node $db $zTab $iNode]
if {$iNode!=1 && [llength $node]==0} { error "No such node: $iNode" }
if {$iNode != 1 && [llength $node]<[rtree_mincells $db $zTab]} {
puts "Node $iNode: Has only [llength $node] cells"
error ""
}
if {$iNode == 1 && [llength $node]==1 && [rtree_depth $db $zTab]>0} {
set depth [rtree_depth $db $zTab]
puts "Node $iNode: Has only 1 child (tree depth is $depth)"
error ""
}
set nDim [expr {([llength [lindex $node 0]]-1)/2}]
if {$iDepth > 0} {
set d [expr $iDepth-1]
foreach cell $node {
set shouldbe [rtree_node_check $db $zTab [lindex $cell 0] $d]
if {$cell ne $shouldbe} {
puts "Node $iNode: Cell is: {$cell}, should be {$shouldbe}"
error ""
}
}
}
set mapping_table "${zTab}_parent"
set mapping_sql "SELECT parentnode FROM $mapping_table WHERE rowid = \$rowid"
if {$iDepth==0} {
set mapping_table "${zTab}_rowid"
set mapping_sql "SELECT nodeno FROM $mapping_table WHERE rowid = \$rowid"
}
foreach cell $node {
set rowid [lindex $cell 0]
set mapping [db one $mapping_sql]
if {$mapping != $iNode} {
puts "Node $iNode: $mapping_table entry for cell $rowid is $mapping"
error ""
}
}
set ret [list $iNode]
for {set ii 1} {$ii <= $nDim*2} {incr ii} {
set f [lindex $node 0 $ii]
foreach cell $node {
set f2 [lindex $cell $ii]
if {($ii%2)==1 && $f2<$f} {set f $f2}
if {($ii%2)==0 && $f2>$f} {set f $f2}
}
lappend ret $f
}
return $ret
}
proc rtree_dump {db zTab} {
set zRet ""
set nDim [expr {(([llength [$db eval "pragma table_info($zTab)"]]/6)-1)/2}]
set sql "SELECT nodeno, rtreenode($nDim, data) AS node FROM ${zTab}_node"
$db eval $sql {
append zRet [format "% -10s %s\n" $nodeno $node]
}
set zRet
}
proc rtree_nodetreedump {db zTab zIndent iDepth iNode} {
set ret ""
set node [rtree_node $db $zTab $iNode 1]
append ret [format "%-3d %s%s\n" $iNode $zIndent $node]
if {$iDepth>0} {
foreach cell $node {
set i [lindex $cell 0]
append ret [rtree_nodetreedump $db $zTab "$zIndent " [expr $iDepth-1] $i]
}
}
set ret
}
proc rtree_treedump {db zTab} {
set d [rtree_depth $db $zTab]
rtree_nodetreedump $db $zTab "" $d 1
}
-56
View File
@@ -1,56 +0,0 @@
/*
** 2010 August 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
*/
#ifndef _SQLITE3RTREE_H_
#define _SQLITE3RTREE_H_
#include <sqlite3.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry;
/*
** Register a geometry callback named zGeom that can be used as part of an
** R-Tree geometry query as follows:
**
** SELECT ... FROM <rtree> WHERE <rtree col> MATCH $zGeom(... params ...)
*/
int sqlite3_rtree_geometry_callback(
sqlite3 *db,
const char *zGeom,
int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes),
void *pContext
);
/*
** A pointer to a structure of the following type is passed as the first
** argument to callbacks registered using rtree_geometry_callback().
*/
struct sqlite3_rtree_geometry {
void *pContext; /* Copy of pContext passed to s_r_g_c() */
int nParam; /* Size of array aParam[] */
double *aParam; /* Parameters passed to SQL geom function */
void *pUser; /* Callback implementation user data */
void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */
};
#ifdef __cplusplus
} /* end of the 'extern "C"' block */
#endif
#endif /* ifndef _SQLITE3RTREE_H_ */
-50
View File
@@ -1,50 +0,0 @@
# 2008 Sep 08
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing that ticket #3363 is fixed.
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source [file join [file dirname [info script]] rtree_util.tcl]
source $testdir/tester.tcl
ifcapable !rtree {
finish_test
return
}
do_test tkt3363.1.1 {
execsql { CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2) }
} {}
do_test tkt3363.1.2 {
for {set ii 1} {$ii < 50} {incr ii} {
set x 1000000
set y [expr 4000000 + $ii*10]
execsql { INSERT INTO t1 VALUES($ii, $x, $x, $y, $y) }
}
} {}
do_test tkt3363.1.3 {
execsql {
SELECT count(*) FROM t1 WHERE +y2>4000425.0;
}
} {7}
do_test tkt3363.1.4 {
execsql {
SELECT count(*) FROM t1 WHERE y2>4000425.0;
}
} {7}
finish_test
-188
View File
@@ -1,188 +0,0 @@
load ./libsqlite3.dylib
#package require sqlite3
source [file join [file dirname $argv0] rtree_util.tcl]
wm title . "SQLite r-tree viewer"
if {[llength $argv]!=1} {
puts stderr "Usage: $argv0 <database-file>"
puts stderr ""
exit
}
sqlite3 db [lindex $argv 0]
canvas .c -background white -width 400 -height 300 -highlightthickness 0
button .b -text "Parent Node" -command {
set sql "SELECT parentnode FROM $::O(zTab)_parent WHERE nodeno = $::O(iNode)"
set ::O(iNode) [db one $sql]
if {$::O(iNode) eq ""} {set ::O(iNode) 1}
view_node
}
set O(iNode) 1
set O(zTab) ""
set O(listbox_captions) [list]
set O(listbox_itemmap) [list]
set O(listbox_highlight) -1
listbox .l -listvariable ::O(listbox_captions) -yscrollcommand {.ls set}
scrollbar .ls -command {.l yview}
label .status -font courier -anchor w
label .title -anchor w -text "Node 1:" -background white -borderwidth 0
set rtree_tables [list]
db eval {
SELECT name
FROM sqlite_master
WHERE type='table' AND sql LIKE '%virtual%table%using%rtree%'
} {
set nCol [expr [llength [db eval "pragma table_info($name)"]]/6]
if {$nCol != 5} {
puts stderr "Not viewing $name - is not 2-dimensional"
} else {
lappend rtree_tables [list Table $name]
}
}
if {$rtree_tables eq ""} {
puts stderr "Cannot find an r-tree table in database [lindex $argv 0]"
puts stderr ""
exit
}
eval tk_optionMenu .select option_var $rtree_tables
trace add variable option_var write set_option_var
proc set_option_var {args} {
set ::O(zTab) [lindex $::option_var 1]
set ::O(iNode) 1
view_node
}
set ::O(zTab) [lindex $::rtree_tables 0 1]
bind .l <1> {listbox_click [.l nearest %y]}
bind .l <Motion> {listbox_mouseover [.l nearest %y]}
bind .l <Leave> {listbox_mouseover -1}
proc listbox_click {sel} {
if {$sel ne ""} {
set ::O(iNode) [lindex $::O(listbox_captions) $sel 1]
view_node
}
}
proc listbox_mouseover {i} {
set oldid [lindex $::O(listbox_itemmap) $::O(listbox_highlight)]
.c itemconfigure $oldid -fill ""
.l selection clear 0 end
.status configure -text ""
if {$i>=0} {
set id [lindex $::O(listbox_itemmap) $i]
.c itemconfigure $id -fill grey
.c lower $id
set ::O(listbox_highlight) $i
.l selection set $i
.status configure -text [cell_report db $::O(zTab) $::O(iNode) $i]
}
}
grid configure .select -row 0 -column 0 -columnspan 2 -sticky nsew
grid configure .b -row 1 -column 0 -columnspan 2 -sticky nsew
grid configure .l -row 2 -column 0 -sticky nsew
grid configure .status -row 3 -column 0 -columnspan 3 -sticky nsew
grid configure .title -row 0 -column 2 -sticky nsew
grid configure .c -row 1 -column 2 -rowspan 2 -sticky nsew
grid configure .ls -row 2 -column 1 -sticky nsew
grid columnconfigure . 2 -weight 1
grid rowconfigure . 2 -weight 1
proc node_bbox {data} {
set xmin 0
set xmax 0
set ymin 0
set ymax 0
foreach {rowid xmin xmax ymin ymax} [lindex $data 0] break
foreach cell [lrange $data 1 end] {
foreach {rowid x1 x2 y1 y2} $cell break
if {$x1 < $xmin} {set xmin $x1}
if {$x2 > $xmax} {set xmax $x2}
if {$y1 < $ymin} {set ymin $y1}
if {$y2 > $ymax} {set ymax $y2}
}
list $xmin $xmax $ymin $ymax
}
proc view_node {} {
set iNode $::O(iNode)
set zTab $::O(zTab)
set data [rtree_node db $zTab $iNode 12]
set depth [rtree_nodedepth db $zTab $iNode]
.c delete all
set ::O(listbox_captions) [list]
set ::O(listbox_itemmap) [list]
set $::O(listbox_highlight) -1
.b configure -state normal
if {$iNode == 1} {.b configure -state disabled}
.title configure -text "Node $iNode: [cell_report db $zTab $iNode -1]"
foreach {xmin xmax ymin ymax} [node_bbox $data] break
set total_area 0.0
set xscale [expr {double([winfo width .c]-20)/($xmax-$xmin)}]
set yscale [expr {double([winfo height .c]-20)/($ymax-$ymin)}]
set xoff [expr {10.0 - $xmin*$xscale}]
set yoff [expr {10.0 - $ymin*$yscale}]
foreach cell $data {
foreach {rowid x1 x2 y1 y2} $cell break
set total_area [expr {$total_area + ($x2-$x1)*($y2-$y1)}]
set x1 [expr {$x1*$xscale + $xoff}]
set x2 [expr {$x2*$xscale + $xoff}]
set y1 [expr {$y1*$yscale + $yoff}]
set y2 [expr {$y2*$yscale + $yoff}]
set id [.c create rectangle $x1 $y1 $x2 $y2]
if {$depth>0} {
lappend ::O(listbox_captions) "Node $rowid"
lappend ::O(listbox_itemmap) $id
}
}
}
proc cell_report {db zTab iParent iCell} {
set data [rtree_node db $zTab $iParent 12]
set cell [lindex $data $iCell]
foreach {xmin xmax ymin ymax} [node_bbox $data] break
set total_area [expr ($xmax-$xmin)*($ymax-$ymin)]
if {$cell eq ""} {
set cell_area 0.0
foreach cell $data {
foreach {rowid x1 x2 y1 y2} $cell break
set cell_area [expr $cell_area+($x2-$x1)*($y2-$y1)]
}
set cell_area [expr $cell_area/[llength $data]]
set zReport [format "Size = %.1f x %.1f Average child area = %.1f%%" \
[expr $xmax-$xmin] [expr $ymax-$ymin] [expr 100.0*$cell_area/$total_area]\
]
append zReport " Sub-tree height: [rtree_nodedepth db $zTab $iParent]"
} else {
foreach {rowid x1 x2 y1 y2} $cell break
set cell_area [expr ($x2-$x1)*($y2-$y1)]
set zReport [format "Size = %.1f x %.1f Area = %.1f%%" \
[expr $x2-$x1] [expr $y2-$y1] [expr 100.0*$cell_area/$total_area]
]
}
return $zReport
}
view_node
bind .c <Configure> view_node
Executable → Regular
View File
+3328 -5390
View File
File diff suppressed because it is too large Load Diff
+356 -381
View File
@@ -9,6 +9,13 @@
# BCC C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
# USLEEP If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
# THREADSAFE If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
# THREADLIB Specify any extra linker options needed to make the library
# thread safe
#
@@ -35,8 +42,7 @@
# LIBREADLINE Linker options needed by programs using readline() must
# link against.
#
# NAWK Nawk compatible awk program. Older (obsolete?) solaris
# systems need this to avoid using the original AT&T AWK.
# ENCODING "UTF8" or "ISO8859"
#
# Once the macros above are defined, the rest of this make script will
# build the SQLite library and testing tools.
@@ -44,31 +50,19 @@
# This is how we compile
#
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
TCCX += -I$(TOP)/ext/async
TCCX = $(TCC) $(OPTS) $(THREADSAFE) $(USLEEP) -I. -I$(TOP)/src
# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o \
backup.o bitvec.o btmutex.o btree.o build.o \
callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \
fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \
func.o global.o hash.o \
icu.o insert.o journal.o legacy.o loadext.o \
main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
memjournal.o \
mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \
notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \
pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
LIBOBJ+= alter.o analyze.o attach.o auth.o btree.o build.o \
callback.o complete.o date.o delete.o \
expr.o func.o hash.o insert.o \
main.o opcodes.o os_unix.o os_win.o \
pager.o parse.o pragma.o prepare.o printf.o random.o \
select.o table.o tclsqlite.o tokenize.o trigger.o \
update.o util.o vacuum.o \
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbetrace.o \
wal.o walker.o where.o utf.o vtab.o
vdbe.o vdbeapi.o vdbeaux.o vdbefifo.o vdbemem.o \
where.o utf.o legacy.o
# All of the source code files.
#
@@ -77,70 +71,33 @@ SRC = \
$(TOP)/src/analyze.c \
$(TOP)/src/attach.c \
$(TOP)/src/auth.c \
$(TOP)/src/backup.c \
$(TOP)/src/bitvec.c \
$(TOP)/src/btmutex.c \
$(TOP)/src/btree.c \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/build.c \
$(TOP)/src/callback.c \
$(TOP)/src/complete.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/delete.c \
$(TOP)/src/expr.c \
$(TOP)/src/fault.c \
$(TOP)/src/fkey.c \
$(TOP)/src/func.c \
$(TOP)/src/global.c \
$(TOP)/src/hash.c \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/journal.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
$(TOP)/src/malloc.c \
$(TOP)/src/mem0.c \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
$(TOP)/src/mutex_os2.c \
$(TOP)/src/mutex_unix.c \
$(TOP)/src/mutex_w32.c \
$(TOP)/src/notify.c \
$(TOP)/src/os.c \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_os2.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
$(TOP)/src/pager.h \
$(TOP)/src/parse.y \
$(TOP)/src/pcache.c \
$(TOP)/src/pcache.h \
$(TOP)/src/pcache1.c \
$(TOP)/src/pragma.c \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
$(TOP)/src/resolve.c \
$(TOP)/src/rowset.c \
$(TOP)/src/select.c \
$(TOP)/src/status.c \
$(TOP)/src/shell.c \
$(TOP)/src/sqlite.h.in \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/tokenize.c \
@@ -153,212 +110,75 @@ SRC = \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeapi.c \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbefifo.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
$(TOP)/src/wal.c \
$(TOP)/src/wal.h \
$(TOP)/src/walker.c \
$(TOP)/src/where.c
# Source code for extensions
#
SRC += \
$(TOP)/ext/fts1/fts1.c \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.c \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_porter.c \
$(TOP)/ext/fts1/fts1_tokenizer.h \
$(TOP)/ext/fts1/fts1_tokenizer1.c
SRC += \
$(TOP)/ext/fts2/fts2.c \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.c \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_icu.c \
$(TOP)/ext/fts2/fts2_porter.c \
$(TOP)/ext/fts2/fts2_tokenizer.h \
$(TOP)/ext/fts2/fts2_tokenizer.c \
$(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3Int.h \
$(TOP)/ext/fts3/fts3_aux.c \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_hash.c \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_icu.c \
$(TOP)/ext/fts3/fts3_porter.c \
$(TOP)/ext/fts3/fts3_snippet.c \
$(TOP)/ext/fts3/fts3_tokenizer.h \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_tokenizer1.c \
$(TOP)/ext/fts3/fts3_write.c
SRC += \
$(TOP)/ext/icu/sqliteicu.h \
$(TOP)/ext/icu/icu.c
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
# Generated source code files
#
SRC += \
keywordhash.h \
opcodes.c \
opcodes.h \
parse.c \
parse.h \
sqlite3.h
# Source code to the test files.
#
TESTSRC = \
$(TOP)/src/btree.c \
$(TOP)/src/date.c \
$(TOP)/src/func.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
$(TOP)/src/pragma.c \
$(TOP)/src/printf.c \
$(TOP)/src/test1.c \
$(TOP)/src/test2.c \
$(TOP)/src/test3.c \
$(TOP)/src/test4.c \
$(TOP)/src/test5.c \
$(TOP)/src/test6.c \
$(TOP)/src/test7.c \
$(TOP)/src/test8.c \
$(TOP)/src/test9.c \
$(TOP)/src/test_autoext.c \
$(TOP)/src/test_async.c \
$(TOP)/src/test_backup.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_demovfs.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_func.c \
$(TOP)/src/test_fuzzer.c \
$(TOP)/src/test_hexio.c \
$(TOP)/src/test_init.c \
$(TOP)/src/test_intarray.c \
$(TOP)/src/test_journal.c \
$(TOP)/src/test_malloc.c \
$(TOP)/src/test_multiplex.c \
$(TOP)/src/test_mutex.c \
$(TOP)/src/test_onefile.c \
$(TOP)/src/test_osinst.c \
$(TOP)/src/test_pcache.c \
$(TOP)/src/test_quota.c \
$(TOP)/src/test_rtree.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_stat.c \
$(TOP)/src/test_superlock.c \
$(TOP)/src/test_syscall.c \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_wholenumber.c \
$(TOP)/src/test_wsd.c
#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
TESTSRC2 = \
$(TOP)/src/attach.c \
$(TOP)/src/backup.c \
$(TOP)/src/btree.c \
$(TOP)/src/build.c \
$(TOP)/src/date.c \
$(TOP)/src/expr.c \
$(TOP)/src/func.c \
$(TOP)/src/insert.c \
$(TOP)/src/wal.c \
$(TOP)/src/mem5.c \
$(TOP)/src/os.c \
$(TOP)/src/os_os2.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
$(TOP)/src/pragma.c \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
$(TOP)/src/pcache.c \
$(TOP)/src/pcache1.c \
$(TOP)/src/select.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/utf.c \
$(TOP)/src/util.c \
$(TOP)/src/vdbeapi.c \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbe.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/where.c \
parse.c \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3_aux.c \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_write.c \
$(TOP)/ext/async/sqlite3async.c
$(TOP)/src/md5.c \
$(TOP)/src/where.c
# Header files used by all library source files.
#
HDR = \
sqlite3.h \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
keywordhash.h \
$(TOP)/src/mutex.h \
opcodes.h \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/pager.h \
$(TOP)/src/pcache.h \
parse.h \
sqlite3.h \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/os_unix.h \
$(TOP)/src/os_win.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeInt.h
parse.h
# Header files used by extensions
# Header files used by the VDBE submodule
#
EXTHDR += \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3Int.h \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_tokenizer.h
EXTHDR += \
$(TOP)/ext/rtree/rtree.h
EXTHDR += \
$(TOP)/ext/icu/sqliteicu.h
VDBEHDR = \
$(HDR) \
$(TOP)/src/vdbeInt.h
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.a sqlite3$(EXE)
# Generate the file "last_change" which contains the date of change
# of the most recently modified source code file
#
last_change: $(SRC)
cat $(SRC) | grep '$$Id: ' | sort +4 | tail -1 \
| awk '{print $$5,$$6}' >last_change
libsqlite3.a: $(LIBOBJ)
$(AR) libsqlite3.a $(LIBOBJ)
$(RANLIB) libsqlite3.a
sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \
$(TOP)/src/shell.c \
libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(TOP)/src/shell.c \
libsqlite3.a $(LIBREADLINE) $(THREADLIB)
objects: $(LIBOBJ_ORIG)
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
@@ -366,138 +186,162 @@ sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h
# files are automatically generated. This target takes care of
# all that automatic generation.
#
target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl
target_source: $(SRC) $(VDBEHDR) opcodes.c keywordhash.h
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
cp $(SRC) $(VDBEHDR) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
tclsh $(TOP)/tool/vdbe-compress.tcl <tsrc/vdbe.c >vdbe.new
mv vdbe.new tsrc/vdbe.c
touch target_source
sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl
tclsh $(TOP)/tool/mksqlite3c.tcl
echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
cat sqlite3.c >>tclsqlite3.c
echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl
tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl
tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
cp parse.c opcodes.c keywordhash.h tsrc
cp $(TOP)/sqlite3.def tsrc
# Rules to build the LEMON compiler generator
#
lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
lemon: $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
$(BCC) -o lemon $(TOP)/tool/lemon.c
cp $(TOP)/src/lempar.c .
cp $(TOP)/tool/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
# Rules to build individual files
#
# parse.o
# opcodes.o
#
%.o: %.c $(HDR)
$(TCCX) -c $<
alter.o: $(TOP)/src/alter.c $(HDR)
$(TCCX) -c $(TOP)/src/alter.c
# Rules to build individual *.o files from files in the src directory.
#
%.o: $(TOP)/src/%.c $(HDR)
$(TCCX) -c $<
analyze.o: $(TOP)/src/analyze.c $(HDR)
$(TCCX) -c $(TOP)/src/analyze.c
attach.o: $(TOP)/src/attach.c $(HDR)
$(TCCX) -c $(TOP)/src/attach.c
auth.o: $(TOP)/src/auth.c $(HDR)
$(TCCX) -c $(TOP)/src/auth.c
btree.o: $(TOP)/src/btree.c $(HDR) $(TOP)/src/pager.h
$(TCCX) -c $(TOP)/src/btree.c
build.o: $(TOP)/src/build.c $(HDR)
$(TCCX) -c $(TOP)/src/build.c
callback.o: $(TOP)/src/callback.c $(HDR)
$(TCCX) -c $(TOP)/src/callback.c
complete.o: $(TOP)/src/complete.c $(HDR)
$(TCCX) -c $(TOP)/src/complete.c
date.o: $(TOP)/src/date.c $(HDR)
$(TCCX) -c $(TOP)/src/date.c
delete.o: $(TOP)/src/delete.c $(HDR)
$(TCCX) -c $(TOP)/src/delete.c
expr.o: $(TOP)/src/expr.c $(HDR)
$(TCCX) -c $(TOP)/src/expr.c
func.o: $(TOP)/src/func.c $(HDR)
$(TCCX) -c $(TOP)/src/func.c
hash.o: $(TOP)/src/hash.c $(HDR)
$(TCCX) -c $(TOP)/src/hash.c
insert.o: $(TOP)/src/insert.c $(HDR)
$(TCCX) -c $(TOP)/src/insert.c
legacy.o: $(TOP)/src/legacy.c $(HDR)
$(TCCX) -c $(TOP)/src/legacy.c
main.o: $(TOP)/src/main.c $(HDR)
$(TCCX) -c $(TOP)/src/main.c
pager.o: $(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h
$(TCCX) -c $(TOP)/src/pager.c
opcodes.o: opcodes.c
$(TCCX) -c opcodes.c
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
sort -n -b +2 opcodes.h | awk -f $(TOP)/mkopcodec.awk >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | awk -f $(TOP)/mkopcodeh.awk >opcodes.h
os_unix.o: $(TOP)/src/os_unix.c $(HDR)
$(TCCX) -c $(TOP)/src/os_unix.c
os_win.o: $(TOP)/src/os_win.c $(HDR)
$(TCCX) -c $(TOP)/src/os_win.c
parse.o: parse.c $(HDR)
$(TCCX) -c parse.c
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon
cp $(TOP)/src/parse.y .
./lemon $(OPTS) parse.y
pragma.o: $(TOP)/src/pragma.c $(HDR)
$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/pragma.c
prepare.o: $(TOP)/src/prepare.c $(HDR)
$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/prepare.c
printf.o: $(TOP)/src/printf.c $(HDR)
$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/printf.c
random.o: $(TOP)/src/random.c $(HDR)
$(TCCX) -c $(TOP)/src/random.c
select.o: $(TOP)/src/select.c $(HDR)
$(TCCX) -c $(TOP)/src/select.c
sqlite3.h: $(TOP)/src/sqlite.h.in
sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
-e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | awk '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
$(TOP)/src/sqlite.h.in >sqlite3.h
table.o: $(TOP)/src/table.c $(HDR)
$(TCCX) -c $(TOP)/src/table.c
tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR)
$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | \
$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon $(OPTS) parse.y
mv parse.h parse.h.temp
$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
tokenize.o: $(TOP)/src/tokenize.c keywordhash.h $(HDR)
$(TCCX) -c $(TOP)/src/tokenize.c
keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash >keywordhash.h
trigger.o: $(TOP)/src/trigger.c $(HDR)
$(TCCX) -c $(TOP)/src/trigger.c
update.o: $(TOP)/src/update.c $(HDR)
$(TCCX) -c $(TOP)/src/update.c
# Rules to build the extension objects.
#
icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
utf.o: $(TOP)/src/utf.c $(HDR)
$(TCCX) -c $(TOP)/src/utf.c
fts2.o: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
util.o: $(TOP)/src/util.c $(HDR)
$(TCCX) -c $(TOP)/src/util.c
fts2_hash.o: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
vacuum.o: $(TOP)/src/vacuum.c $(HDR)
$(TCCX) -c $(TOP)/src/vacuum.c
fts2_icu.o: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
vdbe.o: $(TOP)/src/vdbe.c $(VDBEHDR)
$(TCCX) -c $(TOP)/src/vdbe.c
fts2_porter.o: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
vdbeapi.o: $(TOP)/src/vdbeapi.c $(VDBEHDR)
$(TCCX) -c $(TOP)/src/vdbeapi.c
fts2_tokenizer.o: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
vdbeaux.o: $(TOP)/src/vdbeaux.c $(VDBEHDR)
$(TCCX) -c $(TOP)/src/vdbeaux.c
fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
vdbefifo.o: $(TOP)/src/vdbefifo.c $(VDBEHDR)
$(TCCX) -c $(TOP)/src/vdbefifo.c
fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c
fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c
fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
vdbemem.o: $(TOP)/src/vdbemem.c $(VDBEHDR)
$(TCCX) -c $(TOP)/src/vdbemem.c
where.o: $(TOP)/src/where.c $(HDR)
$(TCCX) -c $(TOP)/src/where.c
# Rules for building test programs and for running tests
#
@@ -505,49 +349,24 @@ tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a
$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
testfixture$(EXE): $(TOP)/src/tclsqlite.c libsqlite3.a $(TESTSRC)
$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -DSQLITE_TEST=1 -o testfixture$(EXE) \
$(TESTSRC) $(TOP)/src/tclsqlite.c \
libsqlite3.a $(LIBTCL) $(THREADLIB)
# Rules to build the 'testfixture' application.
#
TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
crashtest: $(TOP)/src/tclsqlite.c libsqlite3.a $(TESTSRC) $(TOP)/src/os_test.c
$(TCCX) $(TCL_FLAGS) -DOS_TEST=1 -DTCLSH=1 -DSQLITE_TEST=1 \
-o crashtest \
$(TESTSRC) $(TOP)/src/os_test.c $(TOP)/src/tclsqlite.c \
libsqlite3.a $(LIBTCL) $(THREADLIB)
testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \
-DSQLITE_ENABLE_FTS3=1 \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fulltest: testfixture$(EXE) sqlite3$(EXE)
fulltest: testfixture$(EXE) sqlite3$(EXE) crashtest
./testfixture$(EXE) $(TOP)/test/all.test
soaktest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test -soak=1
test: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/veryquick.test
./testfixture$(EXE) $(TOP)/test/quick.test
# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
#
threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
$(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
-o threadtest3$(EXE) $(THREADLIB)
threadtest: threadtest3$(EXE)
./threadtest3$(EXE)
sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c libsqlite3.a $(TESTSRC) \
$(TOP)/tool/spaceanal.tcl
sed \
-e '/^#/d' \
@@ -556,19 +375,177 @@ sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
-e 's,^,",' \
-e 's,$$,\\n",' \
$(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
$(TCCX) $(TCL_FLAGS) -DTCLSH=2 $(TESTFIXTURE_FLAGS) \
-DSQLITE_TEST=1 -DSQLITE_PRIVATE="" \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o sqlite3_analyzer$(EXE) \
$(LIBTCL) $(THREADLIB)
$(TCCX) $(TCL_FLAGS) -DTCLSH=2 -DSQLITE_TEST=1 -o \
sqlite3_analyzer$(EXE) $(TESTSRC) $(TOP)/src/tclsqlite.c \
libsqlite3.a $(LIBTCL) $(THREADLIB)
TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
# Rules used to build documentation
#
arch.html: $(TOP)/www/arch.tcl
tclsh $(TOP)/www/arch.tcl >arch.html
extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
./testfixture$(EXE) $(TOP)/test/loadext.test
autoinc.html: $(TOP)/www/autoinc.tcl
tclsh $(TOP)/www/autoinc.tcl >autoinc.html
c_interface.html: $(TOP)/www/c_interface.tcl
tclsh $(TOP)/www/c_interface.tcl >c_interface.html
capi3.html: $(TOP)/www/capi3.tcl
tclsh $(TOP)/www/capi3.tcl >capi3.html
capi3ref.html: $(TOP)/www/capi3ref.tcl
tclsh $(TOP)/www/capi3ref.tcl >capi3ref.html
changes.html: $(TOP)/www/changes.tcl
tclsh $(TOP)/www/changes.tcl >changes.html
compile.html: $(TOP)/www/compile.tcl
tclsh $(TOP)/www/compile.tcl >compile.html
copyright.html: $(TOP)/www/copyright.tcl
tclsh $(TOP)/www/copyright.tcl >copyright.html
copyright-release.html: $(TOP)/www/copyright-release.html
cp $(TOP)/www/copyright-release.html .
copyright-release.pdf: $(TOP)/www/copyright-release.pdf
cp $(TOP)/www/copyright-release.pdf .
common.tcl: $(TOP)/www/common.tcl
cp $(TOP)/www/common.tcl .
conflict.html: $(TOP)/www/conflict.tcl
tclsh $(TOP)/www/conflict.tcl >conflict.html
datatypes.html: $(TOP)/www/datatypes.tcl
tclsh $(TOP)/www/datatypes.tcl >datatypes.html
datatype3.html: $(TOP)/www/datatype3.tcl
tclsh $(TOP)/www/datatype3.tcl >datatype3.html
different.html: $(TOP)/www/different.tcl
tclsh $(TOP)/www/different.tcl >different.html
docs.html: $(TOP)/www/docs.tcl
tclsh $(TOP)/www/docs.tcl >docs.html
download.html: $(TOP)/www/download.tcl
mkdir -p doc
tclsh $(TOP)/www/download.tcl >download.html
faq.html: $(TOP)/www/faq.tcl
tclsh $(TOP)/www/faq.tcl >faq.html
fileformat.html: $(TOP)/www/fileformat.tcl
tclsh $(TOP)/www/fileformat.tcl >fileformat.html
formatchng.html: $(TOP)/www/formatchng.tcl
tclsh $(TOP)/www/formatchng.tcl >formatchng.html
index.html: $(TOP)/www/index.tcl last_change
tclsh $(TOP)/www/index.tcl >index.html
lang.html: $(TOP)/www/lang.tcl
tclsh $(TOP)/www/lang.tcl doc >lang.html
pragma.html: $(TOP)/www/pragma.tcl
tclsh $(TOP)/www/pragma.tcl >pragma.html
lockingv3.html: $(TOP)/www/lockingv3.tcl
tclsh $(TOP)/www/lockingv3.tcl >lockingv3.html
mingw.html: $(TOP)/www/mingw.tcl
tclsh $(TOP)/www/mingw.tcl >mingw.html
nulls.html: $(TOP)/www/nulls.tcl
tclsh $(TOP)/www/nulls.tcl >nulls.html
oldnews.html: $(TOP)/www/oldnews.tcl
tclsh $(TOP)/www/oldnews.tcl >oldnews.html
omitted.html: $(TOP)/www/omitted.tcl
tclsh $(TOP)/www/omitted.tcl >omitted.html
opcode.html: $(TOP)/www/opcode.tcl $(TOP)/src/vdbe.c
tclsh $(TOP)/www/opcode.tcl $(TOP)/src/vdbe.c >opcode.html
optimizer.html: $(TOP)/www/optimizer.tcl
tclsh $(TOP)/www/optimizer.tcl >optimizer.html
optoverview.html: $(TOP)/www/optoverview.tcl
tclsh $(TOP)/www/optoverview.tcl >optoverview.html
quickstart.html: $(TOP)/www/quickstart.tcl
tclsh $(TOP)/www/quickstart.tcl >quickstart.html
speed.html: $(TOP)/www/speed.tcl
tclsh $(TOP)/www/speed.tcl >speed.html
sqlite.html: $(TOP)/www/sqlite.tcl
tclsh $(TOP)/www/sqlite.tcl >sqlite.html
support.html: $(TOP)/www/support.tcl
tclsh $(TOP)/www/support.tcl >support.html
tclsqlite.html: $(TOP)/www/tclsqlite.tcl
tclsh $(TOP)/www/tclsqlite.tcl >tclsqlite.html
vdbe.html: $(TOP)/www/vdbe.tcl
tclsh $(TOP)/www/vdbe.tcl >vdbe.html
version3.html: $(TOP)/www/version3.tcl
tclsh $(TOP)/www/version3.tcl >version3.html
whentouse.html: $(TOP)/www/whentouse.tcl
tclsh $(TOP)/www/whentouse.tcl >whentouse.html
# Files to be published on the website.
#
DOC = \
arch.html \
autoinc.html \
c_interface.html \
capi3.html \
capi3ref.html \
changes.html \
compile.html \
copyright.html \
copyright-release.html \
copyright-release.pdf \
conflict.html \
datatypes.html \
datatype3.html \
different.html \
docs.html \
download.html \
faq.html \
fileformat.html \
formatchng.html \
index.html \
lang.html \
lockingv3.html \
mingw.html \
nulls.html \
oldnews.html \
omitted.html \
opcode.html \
optimizer.html \
optoverview.html \
pragma.html \
quickstart.html \
speed.html \
sqlite.html \
support.html \
tclsqlite.html \
vdbe.html \
version3.html \
whentouse.html
doc: common.tcl $(DOC)
mkdir -p doc
mv $(DOC) doc
cp $(TOP)/www/*.gif $(TOP)/art/*.gif doc
# Standard install and cleanup targets
#
@@ -578,10 +555,8 @@ install: sqlite3 libsqlite3.a sqlite3.h
mv sqlite3.h /usr/include
clean:
rm -f *.o sqlite3 libsqlite3.a sqlite3.h opcodes.*
rm -f *.o sqlite3 libsqlite3.a sqlite3.h opcodes.* crashtest
rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
rm -f $(PUBLISH)
rm -f *.da *.bb *.bbg gmon.out
rm -rf tsrc target_source
rm -f testloadext.dll libtestloadext.so
rm -f sqlite3.c fts?amal.c tclsqlite3.c
rm -rf tsrc
+292 -918
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -1 +1 @@
aa90b943254cf348aa2d23ac05e1895b188e2101
e61382aed45a83ad688a0461948f9cd052a527fa

Some files were not shown because too many files have changed in this diff Show More