Compare commits

..

6 Commits

Author SHA1 Message Date
drh 5d6ae89c21 Merge the latest enhancements from trunk.
FossilOrigin-Name: 8498d7727c25859f1a48b85bec44a0b9ad6e8c48112498d6c71882458970c637
2021-07-29 14:27:16 +00:00
drh 68c7fdb7c1 Merge the ".connection" CLI enhancement from trunk.
FossilOrigin-Name: d317d44fbba53883d2b85846d61d4738891bd98f8045ae58e56ddfe6f27ab01a
2021-07-23 19:26:06 +00:00
drh 93dc177980 Merge recent enhancements on trunk into the win-blocking-locks branch.
FossilOrigin-Name: 9ecad2f8c6ee5ce3d8128d646d7de6acb1c8dae1d688bbd0eb3233b364bc9ca4
2021-07-23 14:53:09 +00:00
mistachkin 9513fc97a8 Minor enhancements to new OSTRACE messages.
FossilOrigin-Name: 3249a6a38d9c86d45d5d04b32d552f306d12fd4d3c94d2efed192104c0c3948a
2021-07-19 03:37:09 +00:00
mistachkin 07444ecf8c Initial work on blocking locks for the Win32 VFS. Various other small enhancements.
FossilOrigin-Name: 584c24252a37ef050ced5926a93e52fc47a41d9017b291dc1c1c796712e391d7
2021-07-15 23:12:27 +00:00
drh 6d373c89e0 Enhance the Windows VFS to recognize SQLITE_FCNTL_LOCK_TIMEOUT and make
the value set there available to the winLockFile() function.

FossilOrigin-Name: 954b5d61299d82312a9ca2764f904e2d8318c469f6749d84f8a6871d12a575a8
2021-07-14 19:35:45 +00:00
1198 changed files with 79398 additions and 223172 deletions
+158 -368
View File
@@ -35,7 +35,7 @@ TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
TCC += -I${TOP}/ext/userauth
# Define this for the autoconf-based build, so that the code knows it can
# include the generated sqlite_cfg.h
# include the generated config.h
#
TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
@@ -57,7 +57,6 @@ LIBTCL = @TCL_LIB_SPEC@
#
READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
READLINE_FLAGS += -DHAVE_LINENOISE=@TARGET_HAVE_LINENOISE@
# The library that programs using readline() must link against.
#
@@ -83,13 +82,9 @@ TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
# The same set of OMIT and ENABLE flags should be passed to the
# LEMON parser generator and the mkkeywordhash tool as well.
#
# Add OPTIONS=... on the command line to append additional options
# to the OPT_FEATURE_FLAGS.
#
OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ $(OPTIONS)
OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
TCC += $(OPT_FEATURE_FLAGS)
TCC += $(OPT_FEATURE_FLAGS)
# Add in any optional parameters specified on the make commane line
# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
@@ -121,15 +116,14 @@ HAVE_TCL = @HAVE_TCL@
#
TCLSH_CMD = @TCLSH_CMD@
# Additional options when running tests using testrunner.tcl
# This is usually either blank, or else --status
#
TSTRNNR_OPTS = @TSTRNNR_OPTS@
# Where do we want to install the tcl plugin
#
TCLLIBDIR = @TCLLIBDIR@
# The suffix used on shared libraries. Ex: ".dll", ".so", ".dylib"
#
SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
# If gcov support was enabled by the configure script, add the appropriate
# flags here. It's not always as easy as just having the user add the right
# CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
@@ -186,11 +180,11 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
fts5.lo \
func.lo global.lo hash.lo \
icu.lo insert.lo json.lo legacy.lo loadext.lo \
icu.lo insert.lo json1.lo legacy.lo loadext.lo \
main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
memdb.lo memjournal.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo \
sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
@@ -240,7 +234,6 @@ SRC = \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/json.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
@@ -263,7 +256,6 @@ SRC = \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_setup.h \
$(TOP)/src/os_kv.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/os_win.h \
@@ -321,6 +313,24 @@ SRC = \
# Source code for extensions
#
SRC += \
$(TOP)/ext/fts1/fts1.c \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.c \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_porter.c \
$(TOP)/ext/fts1/fts1_tokenizer.h \
$(TOP)/ext/fts1/fts1_tokenizer1.c
SRC += \
$(TOP)/ext/fts2/fts2.c \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.c \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_icu.c \
$(TOP)/ext/fts2/fts2_porter.c \
$(TOP)/ext/fts2/fts2_tokenizer.h \
$(TOP)/ext/fts2/fts2_tokenizer.c \
$(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3.h \
@@ -356,6 +366,7 @@ SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h \
$(TOP)/ext/rbu/sqlite3rbu.c
SRC += \
$(TOP)/ext/misc/json1.c \
$(TOP)/ext/misc/stmt.c
# Generated source code files
@@ -366,7 +377,7 @@ SRC += \
opcodes.h \
parse.c \
parse.h \
sqlite_cfg.h \
config.h \
shell.c \
sqlite3.h
@@ -379,6 +390,7 @@ TESTSRC = \
$(TOP)/src/test4.c \
$(TOP)/src/test5.c \
$(TOP)/src/test6.c \
$(TOP)/src/test7.c \
$(TOP)/src/test8.c \
$(TOP)/src/test9.c \
$(TOP)/src/test_autoext.c \
@@ -407,6 +419,7 @@ TESTSRC = \
$(TOP)/src/test_quota.c \
$(TOP)/src/test_rtree.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_superlock.c \
$(TOP)/src/test_syscall.c \
$(TOP)/src/test_tclsh.c \
@@ -420,12 +433,7 @@ TESTSRC = \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/ext/session/test_session.c \
$(TOP)/ext/recover/sqlite3recover.c \
$(TOP)/ext/recover/dbdata.c \
$(TOP)/ext/recover/test_recover.c \
$(TOP)/ext/intck/test_intck.c \
$(TOP)/ext/intck/sqlite3intck.c \
$(TOP)/ext/rbu/test_rbu.c
$(TOP)/ext/rbu/test_rbu.c
# Statically linked extensions
#
@@ -434,7 +442,6 @@ TESTSRC += \
$(TOP)/ext/expert/test_expert.c \
$(TOP)/ext/misc/amatch.c \
$(TOP)/ext/misc/appendvfs.c \
$(TOP)/ext/misc/basexx.c \
$(TOP)/ext/misc/carray.c \
$(TOP)/ext/misc/cksumvfs.c \
$(TOP)/ext/misc/closure.c \
@@ -453,19 +460,15 @@ TESTSRC += \
$(TOP)/ext/misc/normalize.c \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/prefixes.c \
$(TOP)/ext/misc/qpvtab.c \
$(TOP)/ext/misc/randomjson.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/remember.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/spellfix.c \
$(TOP)/ext/misc/stmtrand.c \
$(TOP)/ext/misc/totype.c \
$(TOP)/ext/misc/unionvtab.c \
$(TOP)/ext/misc/wholenumber.c \
$(TOP)/ext/misc/zipfile.c \
$(TOP)/ext/userauth/userauth.c \
$(TOP)/ext/rtree/test_rtreedoc.c
$(TOP)/ext/userauth/userauth.c
# Source code to the library files needed by the test fixture
#
@@ -487,7 +490,6 @@ TESTSRC2 = \
$(TOP)/src/main.c \
$(TOP)/src/mem5.c \
$(TOP)/src/os.c \
$(TOP)/src/os_kv.c \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/pager.c \
@@ -499,7 +501,6 @@ TESTSRC2 = \
$(TOP)/src/pcache1.c \
$(TOP)/src/select.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/treeview.c \
$(TOP)/src/utf.c \
$(TOP)/src/util.c \
$(TOP)/src/vdbeapi.c \
@@ -551,10 +552,18 @@ HDR = \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vxworks.h \
$(TOP)/src/whereInt.h \
sqlite_cfg.h
config.h
# Header files used by extensions
#
EXTHDR += \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3Int.h \
@@ -598,8 +607,7 @@ TESTOPTS = --verbose=file --output=test-out.txt
# Extra compiler options for various shell tools
#
SHELL_OPT += -DSQLITE_DQS=0
SHELL_OPT += -DSQLITE_ENABLE_FTS4
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
#SHELL_OPT += -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_RTREE
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
@@ -609,68 +617,24 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1
FUZZERSHELL_OPT =
FUZZCHECK_OPT += -I$(TOP)/test
FUZZCHECK_OPT += -I$(TOP)/ext/recover
FUZZCHECK_OPT += \
-DSQLITE_OSS_FUZZ \
-DSQLITE_ENABLE_BYTECODE_VTAB \
-DSQLITE_ENABLE_DBPAGE_VTAB \
-DSQLITE_ENABLE_DBSTAT_VTAB \
-DSQLITE_ENABLE_BYTECODE_VTAB \
-DSQLITE_ENABLE_DESERIALIZE \
-DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-DSQLITE_ENABLE_FTS3_PARENTHESIS \
-DSQLITE_ENABLE_FTS4 \
-DSQLITE_ENABLE_FTS5 \
-DSQLITE_ENABLE_GEOPOLY \
-DSQLITE_ENABLE_MATH_FUNCTIONS \
-DSQLITE_ENABLE_MEMSYS5 \
-DSQLITE_ENABLE_NORMALIZE \
-DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-DSQLITE_ENABLE_PREUPDATE_HOOK \
-DSQLITE_ENABLE_RTREE \
-DSQLITE_ENABLE_SESSION \
-DSQLITE_ENABLE_STMTVTAB \
-DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \
-DSQLITE_ENABLE_STAT4 \
-DSQLITE_ENABLE_STMT_SCANSTATUS \
-DSQLITE_MAX_MEMORY=50000000 \
-DSQLITE_MAX_MMAP_SIZE=0 \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_PRINTF_PRECISION_LIMIT=1000 \
-DSQLITE_PRIVATE="" \
-DSQLITE_STRICT_SUBTYPE=1 \
-DSQLITE_STATIC_RANDOMJSON
FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c
FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c
FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
FUZZCHECK_SRC += $(TOP)/test/vt02.c
FUZZCHECK_SRC += $(TOP)/ext/misc/percentile.c
FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS
#FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5
FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT =
ST_OPT = -DSQLITE_OS_KV_OPTIONAL
# In wasi-sdk builds, disable the CLI shell build in the "all" target.
SQLITE3_SHELL_TARGET_ = sqlite3$(TEXE)
SQLITE3_SHELL_TARGET_1 =
SQLITE3_SHELL_TARGET = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@)
# Use $(libtclsqlite3.la_$(HAVE_TCL)) to resolve to either
# libtclsqlite3.la or an empty value.
libtclsqlite3.la_0 =
libtclsqlite3.la_1 = libtclsqlite3.la
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.la $(SQLITE3_SHELL_TARGET) \
$(libtclsqlite3.la_$(HAVE_TCL))
all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
Makefile: $(TOP)/Makefile.in
./config.status
@@ -694,26 +658,12 @@ sqlite3$(TEXE): shell.c sqlite3.c
shell.c sqlite3.c \
$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.lo sqlite3.h
$(LTLINK) -I$(TOP)/ext/misc -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
RSYNC_SRC = \
$(TOP)/tool/sqlite3-rsync.c \
sqlite3.c
RSYNC_OPT = \
-DSQLITE_ENABLE_DBPAGE_VTAB \
-USQLITE_THREADSAFE \
-DSQLITE_THREADSAFE=0 \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_OMIT_DEPRECATED
sqlite3-rsync$(TEXE): $(RSYNC_SRC)
$(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS)
scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo
$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
@@ -724,40 +674,13 @@ srcck1$(BEXE): $(TOP)/tool/srcck1.c
sourcetest: srcck1$(BEXE) sqlite3.c
./srcck1 sqlite3.c
src-verify: $(TOP)/tool/src-verify.c
$(BCC) -o src-verify$(BEXE) $(TOP)/tool/src-verify.c
verify-source: ./src-verify
./src-verify $(TOP)
fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
fuzzcheck-asan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
$(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
fuzzcheck-ubsan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
# Usage: FUZZDB=filename make run-fuzzcheck
#
# Where filename is a fuzzcheck database, this target builds and runs
# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
#
# FUZZDB can be a glob pattern of two or more databases. Example:
#
# FUZZDB=test/fuzzdata*.db make run-fuzzcheck
#
run-fuzzcheck: fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
./fuzzcheck$(TEXE) --spinner $(FUZZDB)
./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
$(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -817,26 +740,13 @@ mptest: mptester$(TEXE)
$(MPTEST2) --journalmode DELETE
has_tclsh84:
sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD)
touch has_tclsh84
has_tclsh85:
sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
touch has_tclsh85
has_tclconfig:
@ if test x"$(HAVE_TCL)" != "x1"; then echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi
touch has_tclconfig
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
# build on the target system. Some of the C source code and header
# files are automatically generated. This target takes care of
# all that automatic generation.
#
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl has_tclsh84 fts5.c
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
@@ -846,20 +756,11 @@ has_tclconfig:
cp fts5.c fts5.h tsrc
touch .target_source
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC)
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS)
cp tsrc/sqlite3ext.h .
cp $(TOP)/ext/session/sqlite3session.h .
sqlite3r.h: sqlite3.h has_tclsh84
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h
sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84
cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
cp $(TOP)/ext/recover/dbdata.c tsrc/
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC)
sqlite3ext.h: .target_source
cp tsrc/sqlite3ext.h .
@@ -869,7 +770,7 @@ tclsqlite3.c: sqlite3.c
echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl has_tclsh84
sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
# Rule to build the amalgamation
@@ -971,9 +872,6 @@ hash.lo: $(TOP)/src/hash.c $(HDR)
insert.lo: $(TOP)/src/insert.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c
json.lo: $(TOP)/src/json.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/json.c
legacy.lo: $(TOP)/src/legacy.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c
@@ -1034,9 +932,6 @@ pcache1.lo: $(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h
os.lo: $(TOP)/src/os.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c
os_kv.lo: $(TOP)/src/os_kv.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_kv.c
os_unix.lo: $(TOP)/src/os_unix.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_unix.c
@@ -1151,16 +1046,16 @@ tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR)
tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR)
$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
tclsqlite3$(TEXE): has_tclconfig tclsqlite-shell.lo libsqlite3.la
tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la
$(LTLINK) -o $@ tclsqlite-shell.lo \
libsqlite3.la $(LIBTCL)
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl has_tclsh84
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl
$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl has_tclsh84
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
@@ -1171,10 +1066,10 @@ parse.c: $(TOP)/src/parse.y lemon$(BEXE)
cp $(TOP)/src/parse.y .
./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION has_tclsh84
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION has_tclsh84
sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION
echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@
@@ -1184,40 +1079,26 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash$(BEXE) >keywordhash.h
# Source and header files that shell.c depends on
SHELL_DEP = \
$(TOP)/src/shell.c.in \
$(TOP)/ext/expert/sqlite3expert.c \
$(TOP)/ext/expert/sqlite3expert.h \
$(TOP)/ext/intck/sqlite3intck.c \
$(TOP)/ext/intck/sqlite3intck.h \
$(TOP)/ext/misc/appendvfs.c \
$(TOP)/ext/misc/base64.c \
$(TOP)/ext/misc/base85.c \
$(TOP)/ext/misc/completion.c \
$(TOP)/ext/misc/decimal.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/ieee754.c \
$(TOP)/ext/misc/memtrace.c \
$(TOP)/ext/misc/pcachetrace.c \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/sha1.c \
$(TOP)/ext/misc/shathree.c \
$(TOP)/ext/misc/sqlar.c \
$(TOP)/ext/misc/sqlite3_stdio.c \
$(TOP)/ext/misc/sqlite3_stdio.h \
$(TOP)/ext/misc/uint.c \
$(TOP)/ext/misc/vfstrace.c \
$(TOP)/ext/misc/zipfile.c \
$(TOP)/ext/recover/dbdata.c \
$(TOP)/ext/recover/sqlite3recover.c \
$(TOP)/ext/recover/sqlite3recover.h \
$(TOP)/src/test_windirent.c \
$(TOP)/src/test_windirent.h
# Source files that go into making shell.c
SHELL_SRC = \
$(TOP)/src/shell.c.in \
$(TOP)/ext/misc/appendvfs.c \
$(TOP)/ext/misc/completion.c \
$(TOP)/ext/misc/decimal.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/ieee754.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/shathree.c \
$(TOP)/ext/misc/sqlar.c \
$(TOP)/ext/misc/uint.c \
$(TOP)/ext/expert/sqlite3expert.c \
$(TOP)/ext/expert/sqlite3expert.h \
$(TOP)/ext/misc/zipfile.c \
$(TOP)/ext/misc/memtrace.c \
$(TOP)/src/test_windirent.c
shell.c: $(SHELL_DEP) $(TOP)/tool/mkshellc.tcl has_tclsh84
shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
@@ -1228,6 +1109,24 @@ shell.c: $(SHELL_DEP) $(TOP)/tool/mkshellc.tcl has_tclsh84
icu.lo: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
fts2.lo: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
fts2_hash.lo: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
fts2_icu.lo: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
fts2_porter.lo: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
fts2_tokenizer.lo: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
fts2_tokenizer1.lo: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
fts3.lo: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
@@ -1276,6 +1175,9 @@ userauth.lo: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
json1.lo: $(TOP)/ext/misc/json1.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
stmt.lo: $(TOP)/ext/misc/stmt.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
@@ -1305,7 +1207,7 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)
fts5parse.h: fts5parse.c
fts5.c: $(FTS5_SRC) has_tclsh84
fts5.c: $(FTS5_SRC)
$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
@@ -1333,27 +1235,22 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB
TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC
TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON
TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1
TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
TESTFIXTURE_SRC1 = sqlite3.c
TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
testfixture$(TEXE): has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC)
testfixture$(TEXE): $(TESTFIXTURE_SRC)
$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
coretestprogs: testfixture$(BEXE) sqlite3$(BEXE)
coretestprogs: $(TESTPROGS)
testprogs: $(TESTPROGS) srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
testprogs: coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
# A very detailed test running most or all test cases
fulltest: alltest fuzztest
# Run most or all tcl test cases
alltest: $(TESTPROGS)
fulltest: $(TESTPROGS) fuzztest
./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
# Really really long testing
@@ -1365,17 +1262,11 @@ fulltestonly: $(TESTPROGS) fuzztest
./testfixture$(TEXE) $(TOP)/test/full.test
# Fuzz testing
#
# WARNING: When the "fuzztest" target is run by the testrunner.tcl script,
# it does not actually run this code. Instead, it schedules equivalent
# commands. Therefore, if this target is updated, then code in
# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated.
#
fuzztest: fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE)
fuzztest: fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
./fuzzcheck$(TEXE) $(FUZZDATA)
./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE)
valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA)
valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
@@ -1384,50 +1275,15 @@ valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE)
tcltest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
# Runs all the same tests cases as the "tcltest" target but uses
# the testrunner.tcl script to run them in multiple cores
# concurrently.
testrunner: testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/testrunner.tcl
# This is the testing target preferred by the core SQLite developers.
# It runs tests under a standard configuration, regardless of how
# ./configure was run. The devs run "make devtest" prior to each
# check-in, at a minimum. Probably other tests too, but at least this
# one.
#
devtest: srctree-check sourcetest
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS)
mdevtest: srctree-check has_tclsh85
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS)
sdevtest: has_tclsh85
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS)
# Validate that various generated files in the source tree
# are up-to-date.
#
srctree-check: $(TOP)/tool/srctree-check.tcl
$(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl
# Testing for a release
#
releasetest: srctree-check has_tclsh85 verify-source
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS)
# Minimal testing that runs in less than 3 minutes
#
quicktest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
# Try to run tests on whatever options are specified by the
# ./configure. The developers seldom use this target. Instead
# they use "make devtest" which runs tests on a standard set of
# options regardless of how SQLite is configured. This "test"
# target is provided for legacy only.
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest
test: fuzztest sourcetest $(TESTPROGS) tcltest
# Run a test using valgrind. This can take a really long time
# because valgrind is so much slower than a native machine.
@@ -1442,19 +1298,19 @@ valgrindtest: $(TESTPROGS) valgrindfuzz
smoketest: $(TESTPROGS) fuzzcheck$(TEXE)
./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
shelltest:
$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell
shelltest: $(TESTPROGS)
./testfixture$(TEXT) $(TOP)/test/permutations.test shell
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c
sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
sqltclsh$(TEXE): has_tclconfig sqltclsh.c
sqltclsh$(TEXE): sqltclsh.c
$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
@@ -1470,10 +1326,10 @@ CHECKER_DEPS =\
$(TOP)/ext/misc/btreeinfo.c \
$(TOP)/ext/repair/sqlite3_checker.c.in
sqlite3_checker.c: $(CHECKER_DEPS) has_tclsh85
sqlite3_checker.c: $(CHECKER_DEPS)
$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
sqlite3_checker$(TEXE): has_tclconfig sqlite3_checker.c
sqlite3_checker$(TEXE): sqlite3_checker.c
$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
@@ -1519,7 +1375,7 @@ LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h
wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c Makefile
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c
$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
startup$(TEXE): $(TOP)/test/startup.c sqlite3.c
@@ -1556,11 +1412,6 @@ amalgamation-tarball: sqlite3.c sqlite3rc.h
snapshot-tarball: sqlite3.c sqlite3rc.h
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
# Build a ZIP archive containing various command-line tools.
#
tool-zip: testfixture sqlite3 sqldiff sqlite3_analyzer $(TOP)/tool/mktoolzip.tcl
./testfixture $(TOP)/tool/mktoolzip.tcl
# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
@@ -1581,18 +1432,16 @@ threadtest: threadtest3$(TEXE)
threadtest5: sqlite3.c $(TOP)/test/threadtest5.c
$(LTLINK) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(TLIBS)
releasetest:
$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
# Standard install and cleanup targets
#
lib_install: libsqlite3.la
$(INSTALL) -d $(DESTDIR)$(libdir)
$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
# Use $(tcl_install_$(HAVE_TCL)) to resolve to either tcl_install or
# an empty value.
tcl_install_0 =
tcl_install_1 = tcl_install
install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc $(tcl_install_$(HAVE_TCL))
install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
$(INSTALL) -d $(DESTDIR)$(bindir)
$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
$(INSTALL) -d $(DESTDIR)$(includedir)
@@ -1603,71 +1452,43 @@ install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc $(tcl_install_$(HAVE_TC
pkgIndex.tcl:
echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@
tcl_install: lib_install libtclsqlite3.la pkgIndex.tcl
$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
# Build the SQLite TCL extension in a way that make it compatible
# with whatever version of TCL is running as $TCLSH_CMD, possibly defined
# by --with-tclsh=
#
tclextension: tclsqlite3.c
$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD
# to find it.
#
tclextension-install: tclsqlite3.c
$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
# Install the SQLite TCL extension that is used by $TCLSH_CMD
#
tclextension-uninstall:
$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall
# List all installed the SQLite TCL extension that is are accessible
# by $TCLSH_CMD, included prior versions.
#
tclextension-list:
$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info
# Remove build products sufficient so that subsequent makes will recompile
# everything from scratch. Do not remove:
#
# * test results and test logs
# * output from ./configure
#
tidy:
rm -f *.lo *.la *.o *.c *.da *.bb *.bbg gmon.* *.rws sqlite3$(TEXE)
rm -f fts5.h keywordhash.h opcodes.h sqlite3.h sqlite3ext.h sqlite3session.h
rm -rf .libs .deps tsrc .target_source
rm -f lemon$(BEXE) sqlite*.tar.gz
rm -f mkkeywordhash$(BEXE) mksourceid$(BEXE)
rm -f parse.* fts5parse.*
rm -f tclsqlite3$(TEXE) $(TESTPROGS)
clean:
rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
rm -f sqlite3.h opcodes.*
rm -rf .libs .deps
rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
rm -f mkkeywordhash$(BEXE) keywordhash.h
rm -f *.da *.bb *.bbg gmon.out
rm -rf tsrc .target_source
rm -f tclsqlite3$(TEXE)
rm -f testfixture$(TEXE) test.db
rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
rm -f wordcount$(TEXE) changeset$(TEXE) version-info$(TEXE)
rm -f *.dll *.lib *.exp *.def *.pc *.vsix *.so *.dylib pkgIndex.tcl
rm -f sqlite3_analyzer$(TEXE) sqlite3-rsync$(TEXE)
rm -f mptester$(TEXE) rbu$(TEXE) srcck1$(TEXE)
rm -f fuzzershell$(TEXE) fuzzcheck$(TEXE) sqldiff$(TEXE) dbhash$(TEXE)
rm -f threadtest5$(TEXE)
rm -f src-verify has_tclsh*
rm -f wordcount$(TEXE) changeset$(TEXE)
rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
rm -f sqlite3.c
rm -f sqlite3rc.h
rm -f shell.c sqlite3ext.h
rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
rm -f sqlite-*-output.vsix
rm -f mptester mptester.exe
rm -f rbu rbu.exe
rm -f srcck1 srcck1.exe
rm -f fuzzershell fuzzershell.exe
rm -f fuzzcheck fuzzcheck.exe
rm -f sqldiff sqldiff.exe
rm -f dbhash dbhash.exe
rm -f fts5.* fts5parse.*
rm -f threadtest5
# Removes build products and test logs. Retains ./configure outputs.
#
clean: tidy
rm -rf omittest* testrunner* testdir*
# Clean up everything. No exceptions.
#
distclean: clean
rm -f sqlite_cfg.h config.log config.status Makefile $(LIBTOOL)
rm -f config.h config.log config.status libtool Makefile sqlite3.pc
#
# Windows section
@@ -1686,34 +1507,3 @@ sqlite3.def: $(REAL_LIBOBJ)
sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def
$(TCC) -shared -o $@ sqlite3.def \
-Wl,"--strip-all" $(REAL_LIBOBJ)
#
# Fiddle app
#
fiddle: sqlite3.c shell.c
make -C ext/wasm fiddle emcc_opt=-Os
#
# Spell-checking for source comments
# The sources checked are either C sources or C source templates.
# Their comments are extracted and processed through aspell using
# a custom dictionary that contains scads of odd identifiers that
# find their way into the comments.
#
# Currently, this target is setup to be "made" in-tree only.
# The output is ephemeral. Redirect it to guide spelling fixups,
# either to correct spelling or add words to tool/custom.txt.
#
./custom.rws: ./tool/custom.txt
@echo 'Updating custom dictionary from tool/custom.txt'
aspell --lang=en create master ./custom.rws < $<
misspell: ./custom.rws has_tclsh84
$(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in
#
# tool/version-info: a utility for emitting sqlite3 version info
# in various forms.
#
version-info$(TEXE): $(TOP)/tool/version-info.c Makefile sqlite3.h
$(LTLINK) $(ST_OPT) -o $@ $(TOP)/tool/version-info.c
+6
View File
@@ -22,6 +22,12 @@ TOP = ../sqlite
BCC = gcc -g -O0
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
#USLEEP =
USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
+115 -265
View File
@@ -18,13 +18,6 @@ USE_AMALGAMATION = 1
!ENDIF
# <</mark>>
# Optionally set EXTRA_SRC to a list of C files to append to
# the generated sqlite3.c.
#
!IFNDEF EXTRA_SRC
EXTRA_SRC =
!ENDIF
# Set this non-0 to enable full warnings (-W4, etc) when compiling.
#
!IFNDEF USE_FULLWARN
@@ -59,13 +52,6 @@ MINIMAL_AMALGAMATION = 0
USE_STDCALL = 0
!ENDIF
# Use the USE_SEH=0 option on the nmake command line to omit structured
# exception handling (SEH) support. SEH is on by default.
#
!IFNDEF USE_SEH
USE_SEH = 1
!ENDIF
# Set this non-0 to have the shell executable link against the core dynamic
# link library.
#
@@ -232,12 +218,6 @@ WIN32HEAP = 0
OSTRACE = 0
!ENDIF
# enable address sanitizer using ASAN=1 on the command-line.
#
!IFNDEF ASAN
ASAN = 0
!ENDIF
# Set this to one of the following values to enable various debugging
# features. Each level includes the debugging options from the previous
# levels. Currently, the recognized values for DEBUG are:
@@ -379,12 +359,11 @@ SQLITE_TCL_DEP =
# the Windows platform.
#
!IFNDEF OPT_FEATURE_FLAGS
OPT_FEATURE_FLAGS = $(OPT_XTRA)
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
@@ -393,14 +372,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF
# Additional feature-options above and beyond what are normally used can be
# be added using OPTIONS=.... on the command-line. These values are
# appended to the OPT_FEATURE_FLAGS variable.
#
!IFDEF OPTIONS
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS)
!ENDIF
# Should the session extension be enabled? If so, add compilation options
# to enable it.
#
@@ -419,14 +390,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
!ENDIF
# Should structured exception handling (SEH) be enabled for WAL mode in
# the core library? It is on by default. Only omit it if the
# USE_SEH=0 option is provided on the nmake command-line.
#
!IF $(USE_SEH)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
# the Windows 10 platform.
#
@@ -617,17 +580,17 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
# <<mark>>
TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
# <</mark>>
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
# <<mark>>
TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl
# <</mark>>
!ELSE
CORE_CCONV_OPTS =
@@ -915,13 +878,6 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
!ENDIF
!ENDIF
# Address sanitizer if ASAN=1
#
!IF $(ASAN)>0
TCC = $(TCC) /fsanitize=address
!ENDIF
# <<mark>>
# The locations of the Tcl header and library files. Also, the library that
# non-stubs enabled programs using Tcl must link against. These variables
@@ -938,7 +894,7 @@ TCLSUFFIX =
!ENDIF
!IFNDEF TCLDIR
TCLDIR = C:\Tcl
TCLDIR = $(TOP)\compat\tcl
!ENDIF
!IFNDEF TCLINCDIR
@@ -1292,11 +1248,11 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
fts5.lo \
func.lo global.lo hash.lo \
icu.lo insert.lo json.lo legacy.lo loadext.lo \
icu.lo insert.lo json1.lo legacy.lo loadext.lo \
main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
memdb.lo memjournal.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo \
sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
@@ -1359,7 +1315,6 @@ SRC00 = \
$(TOP)\src\global.c \
$(TOP)\src\hash.c \
$(TOP)\src\insert.c \
$(TOP)\src\json.c \
$(TOP)\src\legacy.c \
$(TOP)\src\loadext.c \
$(TOP)\src\main.c \
@@ -1377,7 +1332,6 @@ SRC00 = \
$(TOP)\src\mutex_w32.c \
$(TOP)\src\notify.c \
$(TOP)\src\os.c \
$(TOP)\src\os_kv.c \
$(TOP)\src\os_unix.c \
$(TOP)\src\os_win.c
@@ -1457,6 +1411,20 @@ SRC05 = \
$(TOP)\src\wal.h \
$(TOP)\src\whereInt.h
# Extension source code files, part 1.
#
SRC06 = \
$(TOP)\ext\fts1\fts1.c \
$(TOP)\ext\fts1\fts1_hash.c \
$(TOP)\ext\fts1\fts1_porter.c \
$(TOP)\ext\fts1\fts1_tokenizer1.c \
$(TOP)\ext\fts2\fts2.c \
$(TOP)\ext\fts2\fts2_hash.c \
$(TOP)\ext\fts2\fts2_icu.c \
$(TOP)\ext\fts2\fts2_porter.c \
$(TOP)\ext\fts2\fts2_tokenizer.c \
$(TOP)\ext\fts2\fts2_tokenizer1.c
# Extension source code files, part 2.
#
SRC07 = \
@@ -1477,8 +1445,19 @@ SRC07 = \
$(TOP)\ext\rtree\rtree.c \
$(TOP)\ext\session\sqlite3session.c \
$(TOP)\ext\rbu\sqlite3rbu.c \
$(TOP)\ext\misc\json1.c \
$(TOP)\ext\misc\stmt.c
# Extension header files, part 1.
#
SRC08 = \
$(TOP)\ext\fts1\fts1.h \
$(TOP)\ext\fts1\fts1_hash.h \
$(TOP)\ext\fts1\fts1_tokenizer.h \
$(TOP)\ext\fts2\fts2.h \
$(TOP)\ext\fts2\fts2_hash.h \
$(TOP)\ext\fts2\fts2_tokenizer.h
# Extension header files, part 2.
#
SRC09 = \
@@ -1519,7 +1498,7 @@ SRC12 =
# All source code files.
#
SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC07) $(SRC09) $(SRC10) $(SRC11) $(SRC12)
SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) $(SRC12)
# Source code to the test files.
#
@@ -1530,6 +1509,7 @@ TESTSRC = \
$(TOP)\src\test4.c \
$(TOP)\src\test5.c \
$(TOP)\src\test6.c \
$(TOP)\src\test7.c \
$(TOP)\src\test8.c \
$(TOP)\src\test9.c \
$(TOP)\src\test_autoext.c \
@@ -1558,6 +1538,7 @@ TESTSRC = \
$(TOP)\src\test_quota.c \
$(TOP)\src\test_rtree.c \
$(TOP)\src\test_schema.c \
$(TOP)\src\test_server.c \
$(TOP)\src\test_superlock.c \
$(TOP)\src\test_syscall.c \
$(TOP)\src\test_tclsh.c \
@@ -1571,7 +1552,7 @@ TESTSRC = \
$(TOP)\ext\fts3\fts3_term.c \
$(TOP)\ext\fts3\fts3_test.c \
$(TOP)\ext\rbu\test_rbu.c \
$(TOP)\ext\session\test_session.c
$(TOP)\ext\session\test_session.c
# Statically linked extensions.
#
@@ -1580,7 +1561,6 @@ TESTEXT = \
$(TOP)\ext\expert\test_expert.c \
$(TOP)\ext\misc\amatch.c \
$(TOP)\ext\misc\appendvfs.c \
$(TOP)\ext\misc\basexx.c \
$(TOP)\ext\misc\carray.c \
$(TOP)\ext\misc\cksumvfs.c \
$(TOP)\ext\misc\closure.c \
@@ -1599,22 +1579,14 @@ TESTEXT = \
$(TOP)\ext\misc\normalize.c \
$(TOP)\ext\misc\percentile.c \
$(TOP)\ext\misc\prefixes.c \
$(TOP)\ext\misc\qpvtab.c \
$(TOP)\ext\misc\randomjson.c \
$(TOP)\ext\misc\regexp.c \
$(TOP)\ext\misc\remember.c \
$(TOP)\ext\misc\series.c \
$(TOP)\ext\misc\spellfix.c \
$(TOP)\ext\misc\stmtrand.c \
$(TOP)\ext\misc\totype.c \
$(TOP)\ext\misc\unionvtab.c \
$(TOP)\ext\misc\wholenumber.c \
$(TOP)\ext\rtree\test_rtreedoc.c \
$(TOP)\ext\recover\sqlite3recover.c \
$(TOP)\ext\recover\test_recover.c \
$(TOP)\ext\intck\test_intck.c \
$(TOP)\ext\intck\sqlite3intck.c \
$(TOP)\ext\recover\dbdata.c
fts5.c
# If use of zlib is enabled, add the "zipfile.c" source file.
#
@@ -1628,10 +1600,10 @@ TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
TESTSRC2 = \
$(SRC00) \
$(SRC01) \
$(SRC06) \
$(SRC07) \
$(SRC10) \
$(TOP)\ext\async\sqlite3async.c \
fts5.c
$(TOP)\ext\async\sqlite3async.c
# Header files used by all library source files.
#
@@ -1663,6 +1635,14 @@ HDR = \
# Header files used by extensions
#
EXTHDR = $(EXTHDR) \
$(TOP)\ext\fts1\fts1.h \
$(TOP)\ext\fts1\fts1_hash.h \
$(TOP)\ext\fts1\fts1_tokenizer.h
EXTHDR = $(EXTHDR) \
$(TOP)\ext\fts2\fts2.h \
$(TOP)\ext\fts2\fts2_hash.h \
$(TOP)\ext\fts2\fts2_tokenizer.h
EXTHDR = $(EXTHDR) \
$(TOP)\ext\fts3\fts3.h \
$(TOP)\ext\fts3\fts3Int.h \
@@ -1706,70 +1686,24 @@ FUZZDATA = \
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
!ENDIF
# <<mark>>
# Extra compiler options for various test tools.
#
MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS =
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS
MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE=""
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzcheck.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\ossfuzz.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\percentile.c
FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c
FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
@@ -1830,19 +1764,6 @@ pkgIndex.tcl: $(TOP)\VERSION
$(SQLITE3TCLDLL): libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl
$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
tclextension: $(SQLITE3TCLDLL)
tclextension-install: $(SQLITE3TCLDLL)
$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only
tclextension-uninstall:
$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall
tclextension-list:
$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info
# <</mark>>
$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
@@ -1861,25 +1782,12 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT
/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
# <<mark>>
sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
$(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
RSYNC_SRC = \
$(TOP)\tool\sqlite3-rsync.c \
$(SQLITE3C)
RSYNC_OPT = \
-DSQLITE_ENABLE_DBPAGE_VTAB \
-DSQLITE_THREADSAFE=0 \
-DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_OMIT_DEPRECATED
sqlite3-rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS)
$(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1889,12 +1797,6 @@ srcck1.exe: $(TOP)\tool\srcck1.c
sourcetest: srcck1.exe $(SQLITE3C)
srcck1.exe $(SQLITE3C)
src-verify.exe: $(TOP)\tool\src-verify.c
$(LTLINK) $(NO_WARN) $(TOP)\tool\src-verify.c
verify-source: src-verify.exe
src-verify.exe $(TOP)
fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1902,14 +1804,7 @@ dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
$(LTLINK) /F 8388608 $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
run-fuzzcheck: fuzzcheck.exe fuzzcheck-asan.exe
fuzzcheck --spinner $(FUZZDB)
fuzzcheck-asan --spinner $(FUZZDB)
$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1948,7 +1843,9 @@ mptest: mptester.exe
for %i in ($(SRC03)) do copy /Y %i tsrc
for %i in ($(SRC04)) do copy /Y %i tsrc
for %i in ($(SRC05)) do copy /Y %i tsrc
for %i in ($(SRC06)) do copy /Y %i tsrc
for %i in ($(SRC07)) do copy /Y %i tsrc
for %i in ($(SRC08)) do copy /Y %i tsrc
for %i in ($(SRC09)) do copy /Y %i tsrc
for %i in ($(SRC10)) do copy /Y %i tsrc
for %i in ($(SRC11)) do copy /Y %i tsrc
@@ -1962,8 +1859,8 @@ mptest: mptester.exe
move vdbe.new tsrc\vdbe.c
echo > .target_source
sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-verify.exe
$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) $(EXTRA_SRC)
sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL)
$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
@@ -2091,9 +1988,6 @@ hash.lo: $(TOP)\src\hash.c $(HDR)
insert.lo: $(TOP)\src\insert.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c
json.lo: $(TOP)\src\json.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\json.c
legacy.lo: $(TOP)\src\legacy.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c
@@ -2154,9 +2048,6 @@ pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h
os.lo: $(TOP)\src\os.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os.c
os_kv.lo: $(TOP)\src\os_kv.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_kv.c
os_unix.lo: $(TOP)\src\os_unix.c $(HDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_unix.c
@@ -2313,47 +2204,31 @@ mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c
keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
.\mkkeywordhash.exe > keywordhash.h
# Source and header files that shell.c depends on
SHELL_DEP = \
$(TOP)\src\shell.c.in \
$(TOP)\ext\expert\sqlite3expert.c \
$(TOP)\ext\expert\sqlite3expert.h \
$(TOP)\ext\intck\sqlite3intck.c \
$(TOP)\ext\intck\sqlite3intck.h \
$(TOP)\ext\misc\appendvfs.c \
$(TOP)\ext\misc\base64.c \
$(TOP)\ext\misc\base85.c \
$(TOP)\ext\misc\completion.c \
$(TOP)\ext\misc\decimal.c \
$(TOP)\ext\misc\fileio.c \
$(TOP)\ext\misc\ieee754.c \
$(TOP)\ext\misc\memtrace.c \
$(TOP)\ext\misc\pcachetrace.c \
$(TOP)\ext\misc\percentile.c \
$(TOP)\ext\misc\regexp.c \
$(TOP)\ext\misc\series.c \
$(TOP)\ext\misc\sha1.c \
$(TOP)\ext\misc\shathree.c \
$(TOP)\ext\misc\sqlar.c \
$(TOP)\ext\misc\sqlite3_stdio.c \
$(TOP)\ext\misc\sqlite3_stdio.h \
$(TOP)\ext\misc\uint.c \
$(TOP)\ext\misc\vfstrace.c \
$(TOP)\ext\misc\zipfile.c \
$(TOP)\ext\recover\dbdata.c \
$(TOP)\ext\recover\sqlite3recover.c \
$(TOP)\ext\recover\sqlite3recover.h \
$(TOP)\src\test_windirent.c \
$(TOP)\src\test_windirent.h
# Source files that go into making shell.c
SHELL_SRC = \
$(TOP)\src\shell.c.in \
$(TOP)\ext\misc\appendvfs.c \
$(TOP)\ext\misc\completion.c \
$(TOP)\ext\misc\decimal.c \
$(TOP)\ext\misc\fileio.c \
$(TOP)\ext\misc\ieee754.c \
$(TOP)\ext\misc\regexp.c \
$(TOP)\ext\misc\series.c \
$(TOP)\ext\misc\shathree.c \
$(TOP)\ext\misc\uint.c \
$(TOP)\ext\expert\sqlite3expert.c \
$(TOP)\ext\expert\sqlite3expert.h \
$(TOP)\ext\misc\memtrace.c \
$(TOP)\src\test_windirent.c
# If use of zlib is enabled, add the "zipfile.c" source file.
#
!IF $(USE_ZLIB)!=0
SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\sqlar.c
SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\zipfile.c
SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
!ENDIF
shell.c: $(SHELL_DEP) $(TOP)\tool\mkshellc.tcl
shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
zlib:
@@ -2364,6 +2239,24 @@ zlib:
icu.lo: $(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
fts2.lo: $(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c
fts2_hash.lo: $(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c
fts2_icu.lo: $(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c
fts2_porter.lo: $(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c
fts2_tokenizer.lo: $(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c
fts2_tokenizer1.lo: $(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c
fts3.lo: $(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c
@@ -2403,6 +2296,9 @@ fts3_unicode2.lo: $(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR)
fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
json1.lo: $(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c
stmt.lo: $(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c
@@ -2494,10 +2390,9 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON
TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1
TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
@@ -2537,16 +2432,11 @@ extensiontest: testfixture.exe testloadext.dll
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl
.\testfixture.exe $(TOP)\tool\mktoolzip.tcl
coretestprogs: $(TESTPROGS)
coretestprogs: testfixture.exe sqlite3.exe
testprogs: coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe
fulltest: alltest fuzztest
alltest: $(TESTPROGS)
fulltest: $(TESTPROGS) fuzztest
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
@@ -2565,13 +2455,6 @@ queryplantest: testfixture.exe shell
fuzztest: fuzzcheck.exe
.\fuzzcheck.exe $(FUZZDATA)
# Legacy testing target for third-party integrators. The SQLite
# developers seldom use this target themselves. Instead
# they use "nmake /f Makefile.msc devtest" which runs tests on
# a standard set of options
#
test: $(TESTPROGS) sourcetest fuzztest tcltest
# Minimal testing that runs in less than 3 minutes (on a fast machine)
#
quicktest: testfixture.exe sourcetest
@@ -2581,42 +2464,10 @@ quicktest: testfixture.exe sourcetest
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
# The veryquick.test TCL tests.
#
tcltest: testfixture.exe
test: $(TESTPROGS) sourcetest fuzztest
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
# Runs all the same tests cases as the "tcltest" target but uses
# the testrunner.tcl script to run them in multiple cores
# concurrently.
testrunner: testfixture.exe
.\testfixture.exe $(TOP)\test\testrunner.tcl
# This is the testing target preferred by the core SQLite developers.
# It runs tests under a standard configuration. The devs run
# "nmake /f Makefile.msc devtest" prior to each check-in, at a minimum.
# Probably other tests too, but at least this one.
#
devtest: srctree-check sourcetest
$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest
mdevtest:
$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest
# Validate that various generated files in the source tree
# are up-to-date.
#
srctree-check: $(TOP)\tool\srctree-check.tcl
$(TCLSH_CMD) $(TOP)\tool\srctree-check.tcl
# Testing for a release
#
releasetest:
$(TCLSH_CMD) $(TOP)\test\testrunner.tcl release
smoketest: $(TESTPROGS)
@set PATH=$(LIBTCLPATH);$(PATH)
.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
@@ -2624,7 +2475,7 @@ smoketest: $(TESTPROGS)
shelltest: $(TESTPROGS)
.\testfixture.exe $(TOP)\test\permutations.test shell
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP)
sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS)
@@ -2642,14 +2493,14 @@ sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\exp
$(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
CHECKER_DEPS =\
$(TOP)\tool\mkccode.tcl \
$(TOP)/tool/mkccode.tcl \
sqlite3.c \
$(TOP)\src\tclsqlite.c \
$(TOP)\ext\repair\sqlite3_checker.tcl \
$(TOP)\ext\repair\checkindex.c \
$(TOP)\ext\repair\checkfreelist.c \
$(TOP)\ext\misc\btreeinfo.c \
$(TOP)\ext\repair\sqlite3_checker.c.in
$(TOP)/src/tclsqlite.c \
$(TOP)/ext/repair/sqlite3_checker.tcl \
$(TOP)/ext/repair/checkindex.c \
$(TOP)/ext/repair/checkfreelist.c \
$(TOP)/ext/misc/btreeinfo.c \
$(TOP)/ext/repair/sqlite3_checker.c.in
sqlite3_checker.c: $(CHECKER_DEPS)
$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
@@ -2779,7 +2630,7 @@ clean:
del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
del /Q sqlite3rc.h 2>NUL
del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
del /Q sqlite3_analyzer.exe sqlite3_analyzer.c sqlite3-rsync.exe 2>NUL
del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
del /Q sqlite-*-output.vsix 2>NUL
del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
del /Q sqltclsh.* 2>NUL
@@ -2788,5 +2639,4 @@ clean:
del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL
del /Q fts5.* fts5parse.* 2>NUL
del /Q lsm.h lsm1.c 2>NUL
del /q src-verify.exe 2>NUL
# <</mark>>
+101 -206
View File
@@ -1,26 +1,21 @@
<h1 align="center">SQLite Source Repository</h1>
This repository contains the complete source code for the
[SQLite database engine](https://sqlite.org/), including
many test scripts. However, other test scripts
This repository contains the complete source code for the
[SQLite database engine](https://sqlite.org/). Some test scripts
are also included. However, many other test scripts
and most of the documentation are managed separately.
See the [on-line documentation](https://sqlite.org/) for more information
about what SQLite is and how it works from a user's perspective. This
README file is about the source code that goes into building SQLite,
not about how SQLite is used.
## Version Control
SQLite sources are managed using
[Fossil](https://fossil-scm.org/), a distributed version control system
SQLite sources are managed using the
[Fossil](https://www.fossil-scm.org/), a distributed version control system
that was specifically designed and written to support SQLite development.
The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
If you are reading this on GitHub or some other Git repository or service,
then you are looking at a mirror. The names of check-ins and
other artifacts in a Git mirror are different from the official
names for those objects. The official names for check-ins are
names for those objects. The offical names for check-ins are
found in a footer on the check-in comment for authorized mirrors.
The official check-in name can also be seen in the `manifest.uuid` file
in the root of the tree. Always use the official name, not the
@@ -30,33 +25,12 @@ If you pulled your SQLite source code from a secondary source and want to
verify its integrity, there are hints on how to do that in the
[Verifying Code Authenticity](#vauth) section below.
## Contacting The SQLite Developers
The preferred way to ask questions or make comments about SQLite or to
report bugs against SQLite is to visit the
[SQLite Forum](https://sqlite.org/forum) at <https://sqlite.org/forum/>.
Anonymous postings are permitted.
If you think you have found a bug that has security implications and
you do not want to report it on the public forum, you can send a private
email to drh at sqlite dot org.
## Public Domain
The SQLite source code is in the public domain. See
<https://sqlite.org/copyright.html> for details.
Because SQLite is in the public domain, we do not normally accept pull
requests, because if we did take a pull request, the changes in that
pull request might carry a copyright and the SQLite source code would
then no longer be fully in the public domain.
## Obtaining The SQLite Source Code
## Obtaining The Code
If you do not want to use Fossil, you can download tarballs or ZIP
archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
* Latest trunk check-in as
* Lastest trunk check-in as
[Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
[ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
[SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
@@ -73,26 +47,24 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
then click on the "Tarball" or "ZIP Archive" links on the information
page.
To access sources directly using [Fossil](https://fossil-scm.org/home),
If you do want to use Fossil to check out the source tree,
first install Fossil version 2.0 or later.
Source tarballs and precompiled binaries available at
<https://fossil-scm.org/home/uv/download.html>. Fossil is
a stand-alone program. To install, simply download or build the single
executable file and put that file someplace on your $PATH.
(Source tarballs and precompiled binaries available
[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is
a stand-alone program. To install, simply download or build the single
executable file and put that file someplace on your $PATH.)
Then run commands like this:
mkdir -p ~/sqlite
mkdir -p ~/sqlite ~/Fossils
cd ~/sqlite
fossil open https://sqlite.org/src
fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil
fossil open ~/Fossils/sqlite.fossil
The "fossil open" command will take two or three minutes. Afterwards,
you can do fast, bandwidth-efficient updates to the whatever versions
of SQLite you like. Some examples:
After setting up a repository using the steps above, you can always
update to the lastest version using:
fossil update trunk ;# latest trunk check-in
fossil update release ;# latest official release
fossil update trunk:2024-01-01 ;# First trunk check-in after 2024-01-01
fossil update version-3.39.0 ;# Version 3.39.0
fossil update trunk ;# latest trunk check-in
fossil update release ;# latest official release
Or type "fossil ui" to get a web-based user interface.
@@ -106,42 +78,15 @@ script found at the root of the source tree. Then run "make".
For example:
apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make sqlite3 ;# Builds the "sqlite3" command-line tool
make sqlite3.c ;# Build the "amalgamation" source file
make sqldiff ;# Builds the "sqldiff" command-line tool
# Makefile targets below this point require tcl-dev
make tclextension-install ;# Build and install the SQLite TCL extension
make devtest ;# Run development tests
make releasetest ;# Run full release tests
make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool
tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite"
mkdir bld ;# Build will occur in a sibling directory
cd bld ;# Change to the build directory
../sqlite/configure ;# Run the configure script
make ;# Run the makefile.
make sqlite3.c ;# Build the "amalgamation" source file
make test ;# Run some tests (requires Tcl)
See the makefile for additional targets. For debugging builds, the
core developers typically run "configure" with options like this:
../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g'
For release builds, the core developers usually do:
../sqlite/configure --enable-all
Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or
later. The "tclextension-install" target and the test targets that follow
all require TCL development libraries too. ("apt install tcl-dev"). It is
helpful, but is not required, to install the SQLite TCL extension (the
"tclextension-install" target) prior to running tests. The "releasetest"
target has additional requiremenst, such as "valgrind".
On "make" command-lines, one can add "OPTIONS=..." to specify additional
compile-time options over and above those set by ./configure. For example,
to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say:
./configure --enable-all
make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3
See the makefile for additional targets.
The configure script uses autoconf 2.61 and libtool. If the configure
script does not work out for you, there is a generic makefile named
@@ -149,85 +94,60 @@ script does not work out for you, there is a generic makefile named
can copy and edit to suit your needs. Comments on the generic makefile
show what changes are needed.
## Compiling for Windows Using MSVC
## Using MSVC for Windows systems
On Windows, everything can be compiled with MSVC.
You will also need a working installation of TCL.
See the [compile-for-windows.md](doc/compile-for-windows.md) document for
additional information about how to install MSVC and TCL and configure your
build environment.
On Windows, all applicable build products can be compiled with MSVC.
First open the command prompt window associated with the desired compiler
version (e.g. "Developer Command Prompt for VS2013"). Next, use NMAKE
with the provided "Makefile.msc" to build one of the supported targets.
If you want to run tests, you need to let SQLite know the location of your
TCL library, using a command like this:
For example, from the parent directory of the source subtree named "sqlite":
set TCLDIR=c:\Tcl
mkdir bld
cd bld
nmake /f ..\sqlite\Makefile.msc TOP=..\sqlite
nmake /f ..\sqlite\Makefile.msc sqlite3.c TOP=..\sqlite
nmake /f ..\sqlite\Makefile.msc sqlite3.dll TOP=..\sqlite
nmake /f ..\sqlite\Makefile.msc sqlite3.exe TOP=..\sqlite
nmake /f ..\sqlite\Makefile.msc test TOP=..\sqlite
SQLite uses "tclsh.exe" as part of the build process, and so that
program will need to be somewhere on your %PATH%. SQLite itself
does not contain any TCL code, but it does use TCL to help with the
build process and to run tests. You may need to install TCL development
libraries in order to successfully complete some makefile targets.
It is helpful, but is not required, to install the SQLite TCL extension
(the "tclextension-install" target) prior to running tests.
There are several build options that can be set via the NMAKE command
line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument
to the "sqlite3.dll" command line above. When debugging into the SQLite
code, adding the "DEBUG=1" argument to one of the above command lines is
recommended.
Build using Makefile.msc. Example:
SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
is required by the makefiles (including those for MSVC). SQLite contains
a lot of generated code and Tcl is used to do much of that code generation.
nmake /f Makefile.msc sqlite3.exe
nmake /f Makefile.msc sqlite3.c
nmake /f Makefile.msc sqldiff.exe
# Makefile targets below this point require TCL development libraries
nmake /f Makefile.msc tclextension-install
nmake /f Makefile.msc devtest
nmake /f Makefile.msc releasetest
nmake /f Makefile.msc sqlite3_analyzer.exe
There are many other makefile targets. See comments in Makefile.msc for
details.
## Source Code Tour
As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake
command-line to enable new compile-time options. For example:
Most of the core source files are in the **src/** subdirectory. The
**src/** folder also contains files used to build the "testfixture" test
harness. The names of the source files used by "testfixture" all begin
with "test".
The **src/** also contains the "shell.c" file
which is the main program for the "sqlite3.exe"
[command-line shell](https://sqlite.org/cli.html) and
the "tclsqlite.c" file which implements the
[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
(Historical note: SQLite began as a Tcl
extension and only later escaped to the wild as an independent library.)
nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe
Test scripts and programs are found in the **test/** subdirectory.
Addtional test code is found in other source repositories.
See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
additional information.
## Source Tree Map
The **ext/** subdirectory contains code for extensions. The
Full-text search engine is in **ext/fts3**. The R-Tree engine is in
**ext/rtree**. The **ext/misc** subdirectory contains a number of
smaller, single-file extensions, such as a REGEXP operator.
* **src/** - This directory contains the primary source code for the
SQLite core. For historical reasons, C-code used for testing is
also found here. Source files intended for testing begin with "`test`".
The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface
for SQLite and are also not part of the core.
* **test/** - This directory and its subdirectories contains code used
for testing. Files that end in "`.test`" are TCL scripts that run
tests using an augmented TCL interpreter named "testfixture". Use
a command like "`make testfixture`" (unix) or
"`nmake /f Makefile.msc testfixture.exe`" (windows) to build that
augmented TCL interpreter, then run individual tests using commands like
"`testfixture test/main.test`". This test/ subdirectory also contains
additional C code modules and scripts for other kinds of testing.
* **tool/** - This directory contains programs and scripts used to
build some of the machine-generated code that goes into the SQLite
core, as well as to build and run tests and perform diagnostics.
The source code to [the Lemon parser generator](./doc/lemon.html) is
found here. There are also TCL scripts used to build and/or transform
source code files. For example, the tool/mksqlite3h.tcl script reads
the src/sqlite.h.in file and uses it as a template to construct
the deliverable "sqlite3.h" file that defines the SQLite interface.
* **ext/** - Various extensions to SQLite are found under this
directory. For example, the FTS5 subsystem is in "ext/fts5/".
Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built
into the SQLite amalgamation, but not all of them. The
"ext/misc/" subdirectory contains an assortment of one-file extensions,
many of which are omitted from the SQLite core, but which are included
in the [SQLite CLI](https://sqlite.org/cli.html).
* **doc/** - Some documentation files about SQLite internals are found
here. Note, however, that the primary documentation designed for
application developers and users of SQLite is in a completely separate
repository. Note also that the primary API documentation is derived
from specially constructed comments in the src/sqlite.h.in file.
The **tool/** subdirectory contains various scripts and programs used
for building generated source code files or for testing or for generating
accessory programs such as "sqlite3_analyzer(.exe)".
### Generated Source Code Files
@@ -241,7 +161,7 @@ manually-edited files and automatically-generated files.
The SQLite interface is defined by the **sqlite3.h** header file, which is
generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The
[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
The manifest.uuid file contains the SHA3 hash of the particular check-in
and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file
contains the current SQLite version number. The sqlite3.h header is really
@@ -250,7 +170,7 @@ at just the right spots. Note that comment text in the sqlite3.h file is
used to generate much of the SQLite API documentation. The Tcl scripts
used to generate that documentation are in a separate source repository.
The SQL language parser is **parse.c** which is generated from a grammar in
The SQL language parser is **parse.c** which is generate from a grammar in
the src/parse.y file. The conversion of "parse.y" into "parse.c" is done
by the [lemon](./doc/lemon.html) LALR(1) parser generator. The source code
for lemon is at tool/lemon.c. Lemon uses the tool/lempar.c file as a
@@ -260,7 +180,7 @@ generates parse.c.
The **opcodes.h** header file contains macros that define the numbers
corresponding to opcodes in the "VDBE" virtual machine. The opcodes.h
file is generated by scanning the src/vdbe.c source file. The
file is generated by the scanning the src/vdbe.c source file. The
Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
the **opcodes.c** source file, which contains a reverse mapping from
@@ -308,39 +228,33 @@ individual source file exceeds 32K lines in length.
## How It All Fits Together
SQLite is modular in design.
See the [architectural description](https://www.sqlite.org/arch.html)
See the [architectural description](http://www.sqlite.org/arch.html)
for details. Other documents that are useful in
helping to understand how SQLite works include the
[file format](https://www.sqlite.org/fileformat2.html) description,
the [virtual machine](https://www.sqlite.org/opcode.html) that runs
(helping to understand how SQLite works include the
[file format](http://www.sqlite.org/fileformat2.html) description,
the [virtual machine](http://www.sqlite.org/opcode.html) that runs
prepared statements, the description of
[how transactions work](https://www.sqlite.org/atomiccommit.html), and
the [overview of the query planner](https://www.sqlite.org/optoverview.html).
[how transactions work](http://www.sqlite.org/atomiccommit.html), and
the [overview of the query planner](http://www.sqlite.org/optoverview.html).
Decades of effort have gone into optimizing SQLite, both
Years of effort have gone into optimizating SQLite, both
for small size and high performance. And optimizations tend to result in
complex code. So there is a lot of complexity in the current SQLite
implementation. It will not be the easiest library in the world to hack.
### Key source code files
Key files:
* **sqlite.h.in** - This file defines the public interface to the SQLite
library. Readers will need to be familiar with this interface before
trying to understand how the library works internally. This file is
really a template that is transformed into the "sqlite3.h" deliverable
using a script invoked by the makefile.
trying to understand how the library works internally.
* **sqliteInt.h** - this header file defines many of the data objects
used internally by SQLite. In addition to "sqliteInt.h", some
subsystems inside of sQLite have their own header files. These internal
interfaces are not for use by applications. They can and do change
from one release of SQLite to the next.
subsystems have their own header files.
* **parse.y** - This file describes the LALR(1) grammar that SQLite uses
to parse SQL statements, and the actions that are taken at each step
in the parsing process. The file is processed by the
[Lemon Parser Generator](./doc/lemon.html) to produce the actual C code
used for parsing.
in the parsing process.
* **vdbe.c** - This file implements the virtual machine that runs
prepared statements. There are various helper files whose names
@@ -376,17 +290,15 @@ implementation. It will not be the easiest library in the world to hack.
is not part of the core SQLite library. But as most of the tests in this
repository are written in Tcl, the Tcl language bindings are important.
* **test\*.c** - Files in the src/ folder that begin with "test" go into
* **test*.c** - Files in the src/ folder that begin with "test" go into
building the "testfixture.exe" program. The testfixture.exe program is
an enhanced Tcl shell. The testfixture.exe program runs scripts in the
test/ folder to validate the core SQLite code. The testfixture program
(and some other test programs too) is built and run when you type
(and some other test programs too) is build and run when you type
"make test".
* **VERSION**, **manifest**, and **manifest.uuid** - These files define
the current SQLite version number. The "VERSION" file is human generated,
but the "manifest" and "manifest.uuid" files are automatically generated
by the [Fossil version control system](https://fossil-scm.org/).
* **ext/misc/json1.c** - This file implements the various JSON functions
that are build into SQLite.
There are many other source files. Each has a succinct header comment that
describes its purpose and role within the larger system.
@@ -395,38 +307,21 @@ describes its purpose and role within the larger system.
## Verifying Code Authenticity
The `manifest` file at the root directory of the source tree
contains either a SHA3-256 hash or a SHA1 hash
for every source file in the repository.
The name of the version of the entire source tree is just the
SHA3-256 hash of the `manifest` file itself, possibly with the
last line of that file omitted if the last line begins with
"`# Remove this line`".
The `manifest.uuid` file should contain the SHA3-256 hash of the
contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for
older files) for every source file in the repository.
The SHA3-256 hash of the `manifest`
file itself is the official name of the version of the source tree that you
have. The `manifest.uuid` file should contain the SHA3-256 hash of the
`manifest` file. If all of the above hash comparisons are correct, then
you can be confident that your source tree is authentic and unadulterated.
Details on the format for the `manifest` files are available
[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest).
The process of checking source code authenticity is automated by the
makefile:
> make verify-source
Or on windows:
> nmake /f Makefile.msc verify-source
Using the makefile to verify source integrity is good for detecting
accidental changes to the source tree, but malicious changes could be
hidden by also modifying the makefiles.
The format of the `manifest` file should be mostly self-explanatory, but
if you want details, they are available
[here](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
## Contacts
The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
The main SQLite website is [http://www.sqlite.org/](http://www.sqlite.org/)
with geographically distributed backups at
[https://www2.sqlite.org/](https://www2.sqlite.org) and
[https://www3.sqlite.org/](https://www3.sqlite.org).
Contact the SQLite developers through the
[SQLite Forum](https://sqlite.org/forum/). In an emergency, you
can send private email to the lead developer at drh at sqlite dot org.
[http://www2.sqlite.org/](http://www2.sqlite.org) and
[http://www3.sqlite.org/](http://www3.sqlite.org).
+1 -1
View File
@@ -1 +1 @@
3.47.0
3.37.0
Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.3 KiB

+1 -1
View File
@@ -9,7 +9,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
EXTRA_sqlite3_SOURCES = sqlite3.c
sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
include_HEADERS = sqlite3.h sqlite3ext.h
+5 -53
View File
@@ -18,13 +18,6 @@
TOP = .
# Optionally set EXTRA_SRC to a list of C files to append to
# the generated sqlite3.c.
#
!IFNDEF EXTRA_SRC
EXTRA_SRC =
!ENDIF
# Set this non-0 to enable full warnings (-W4, etc) when compiling.
#
!IFNDEF USE_FULLWARN
@@ -59,13 +52,6 @@ MINIMAL_AMALGAMATION = 0
USE_STDCALL = 0
!ENDIF
# Use the USE_SEH=0 option on the nmake command line to omit structured
# exception handling (SEH) support. SEH is on by default.
#
!IFNDEF USE_SEH
USE_SEH = 1
!ENDIF
# Set this non-0 to have the shell executable link against the core dynamic
# link library.
#
@@ -194,12 +180,6 @@ WIN32HEAP = 0
OSTRACE = 0
!ENDIF
# enable address sanitizer using ASAN=1 on the command-line.
#
!IFNDEF ASAN
ASAN = 0
!ENDIF
# Set this to one of the following values to enable various debugging
# features. Each level includes the debugging options from the previous
# levels. Currently, the recognized values for DEBUG are:
@@ -301,12 +281,11 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
# the Windows platform.
#
!IFNDEF OPT_FEATURE_FLAGS
OPT_FEATURE_FLAGS = $(OPT_XTRA)
!IF $(MINIMAL_AMALGAMATION)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
@@ -315,14 +294,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF
# Additional feature-options above and beyond what are normally used can be
# be added using OPTIONS=.... on the command-line. These values are
# appended to the OPT_FEATURE_FLAGS variable.
#
!IFDEF OPTIONS
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS)
!ENDIF
# Should the session extension be enabled? If so, add compilation options
# to enable it.
#
@@ -341,14 +312,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
!ENDIF
# Should structured exception handling (SEH) be enabled for WAL mode in
# the core library? It is on by default. Only omit it if the
# USE_SEH=0 option is provided on the nmake command-line.
#
!IF $(USE_SEH)==0
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
# the Windows 10 platform.
#
@@ -539,12 +502,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
@@ -756,13 +719,6 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
!ENDIF
# Address sanitizer if ASAN=1
#
!IF $(ASAN)>0
TCC = $(TCC) /fsanitize=address
!ENDIF
# Compiler options needed for programs that use the readline() library.
#
!IFNDEF READLINE_FLAGS
@@ -1000,13 +956,9 @@ LIBRESOBJS =
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
!ENDIF
+1 -1
View File
@@ -105,7 +105,7 @@ may be specified in this manner as some require the amalgamation to be built
with them enabled (see http://www.sqlite.org/compile.html). For example, the
following will work:
"OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_JSON=1"
"OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
However, the following will not compile unless the amalgamation was built
with it enabled:
+15
View File
@@ -173,6 +173,21 @@ else
fi
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-json1
#
AC_ARG_ENABLE(json1, [AS_HELP_STRING(
[--enable-json1], [include json1 support [default=yes]])],
[],[enable_json1=yes])
AC_MSG_CHECKING([JSON functions])
if test x"$enable_json1" = "xyes"; then
BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
AC_MSG_RESULT([enabled])
else
AC_MSG_RESULT([disabled])
fi
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-rtree
#
+74 -109
View File
@@ -11,6 +11,8 @@
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: Makefile.in,v 1.59 2005/07/26 19:17:02 mdejong Exp $
#========================================================================
# Add additional lines to handle any additional AC_SUBST cases that
@@ -58,8 +60,6 @@ PKG_HEADERS = @PKG_HEADERS@
#========================================================================
PKG_LIB_FILE = @PKG_LIB_FILE@
PKG_LIB_FILE8 = @PKG_LIB_FILE8@
PKG_LIB_FILE9 = @PKG_LIB_FILE9@
PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@
lib_BINARIES = $(PKG_LIB_FILE)
@@ -73,11 +73,10 @@ exec_prefix = @exec_prefix@
bindir = @bindir@
libdir = @libdir@
includedir = @includedir@
datarootdir = @datarootdir@
runstatedir = @runstatedir@
datadir = @datadir@
mandir = @mandir@
includedir = @includedir@
DESTDIR =
@@ -86,25 +85,24 @@ pkgdatadir = $(datadir)/$(PKG_DIR)
pkglibdir = $(libdir)/$(PKG_DIR)
pkgincludedir = $(includedir)/$(PKG_DIR)
top_builddir = @abs_top_builddir@
top_builddir = .
INSTALL_OPTIONS =
INSTALL = @INSTALL@ $(INSTALL_OPTIONS)
INSTALL_DATA_DIR = @INSTALL_DATA_DIR@
INSTALL_DATA = @INSTALL_DATA@
INSTALL = @INSTALL@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_SCRIPT = @INSTALL_SCRIPT@
INSTALL_LIBRARY = @INSTALL_LIBRARY@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
CC = @CC@
CCLD = @CCLD@
CFLAGS_DEFAULT = @CFLAGS_DEFAULT@
CFLAGS_WARNING = @CFLAGS_WARNING@
CLEANFILES = @CLEANFILES@
EXEEXT = @EXEEXT@
LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@
MAKE_LIB = @MAKE_LIB@
MAKE_SHARED_LIB = @MAKE_SHARED_LIB@
MAKE_STATIC_LIB = @MAKE_STATIC_LIB@
MAKE_STUB_LIB = @MAKE_STUB_LIB@
OBJEXT = @OBJEXT@
RANLIB = @RANLIB@
@@ -119,6 +117,8 @@ TCL_SRC_DIR = @TCL_SRC_DIR@
#TK_BIN_DIR = @TK_BIN_DIR@
#TK_SRC_DIR = @TK_SRC_DIR@
# This is no longer necessary even for packages that use private Tcl headers
#TCL_TOP_DIR_NATIVE = @TCL_TOP_DIR_NATIVE@
# Not used, but retained for reference of what libs Tcl required
#TCL_LIBS = @TCL_LIBS@
@@ -132,52 +132,41 @@ TCL_SRC_DIR = @TCL_SRC_DIR@
EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR)
#EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR)
TCLLIBPATH = $(top_builddir)
TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library`
PKG_ENV = @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \
TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` \
@LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \
PATH="$(EXTRA_PATH):$(PATH)" \
TCLLIBPATH="$(TCLLIBPATH)"
# TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library`
TCLSH_PROG = @TCLSH_PROG@
TCLSH = $(TCLSH_ENV) $(PKG_ENV) $(TCLSH_PROG)
TCLSH = $(TCLSH_ENV) $(TCLSH_PROG)
#WISH_ENV = TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library`
#WISH_PROG = @WISH_PROG@
#WISH = $(TCLSH_ENV) $(WISH_ENV) $(PKG_ENV) $(WISH_PROG)
#WISH = $(TCLSH_ENV) $(WISH_PROG)
SHARED_BUILD = @SHARED_BUILD@
INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I. -I$(srcdir)/..
INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I$(srcdir)/..
#INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@
PKG_CFLAGS = @PKG_CFLAGS@
# TCL_DEFS is not strictly need here, but if you remove it, then you
# must make sure that configure.ac checks for the necessary components
# must make sure that configure.in checks for the necessary components
# that your library may use. TCL_DEFS can actually be a problem if
# you do not compile with a similar machine setup as the Tcl core was
# compiled with.
#DEFS = $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS)
DEFS = @DEFS@ $(PKG_CFLAGS)
# Move pkgIndex.tcl to 'BINARIES' var if it is generated in the Makefile
CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl
CLEANFILES = @CLEANFILES@
CPPFLAGS = @CPPFLAGS@
LIBS = @PKG_LIBS@ @LIBS@
AR = @AR@
CFLAGS = @CFLAGS@
LDFLAGS = @LDFLAGS@
LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@
COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) \
$(CFLAGS_DEFAULT) $(CFLAGS_WARNING) $(SHLIB_CFLAGS) $(CFLAGS)
GDB = gdb
VALGRIND = valgrind
VALGRINDARGS = --tool=memcheck --num-callers=8 --leak-resolution=high \
--leak-check=yes --show-reachable=yes -v
.SUFFIXES: .c .$(OBJEXT)
COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
#========================================================================
# Start of user-definable TARGETS section
@@ -185,7 +174,7 @@ VALGRINDARGS = --tool=memcheck --num-callers=8 --leak-resolution=high \
#========================================================================
# TEA TARGETS. Please note that the "libraries:" target refers to platform
# independent files, and the "binaries:" target includes executable programs and
# independent files, and the "binaries:" target inclues executable programs and
# platform-dependent libraries. Modify these targets so that they install
# the various pieces of your package. The make and install rules
# for the BINARIES that you specified above have already been done.
@@ -204,6 +193,7 @@ binaries: $(BINARIES)
libraries:
#========================================================================
# Your doc target should differentiate from doc builds (by the developer)
# and doc installs (see install-doc), which just install the docs on the
@@ -226,11 +216,11 @@ install-binaries: binaries install-lib-binaries install-bin-binaries
#========================================================================
install-libraries: libraries
@$(INSTALL_DATA_DIR) "$(DESTDIR)$(includedir)"
@mkdir -p $(DESTDIR)$(includedir)
@echo "Installing header files in $(DESTDIR)$(includedir)"
@list='$(PKG_HEADERS)'; for i in $$list; do \
echo "Installing $(srcdir)/$$i" ; \
$(INSTALL_DATA) $(srcdir)/$$i "$(DESTDIR)$(includedir)" ; \
$(INSTALL_DATA) $(srcdir)/$$i $(DESTDIR)$(includedir) ; \
done;
#========================================================================
@@ -239,11 +229,12 @@ install-libraries: libraries
#========================================================================
install-doc: doc
@$(INSTALL_DATA_DIR) "$(DESTDIR)$(mandir)/mann"
@mkdir -p $(DESTDIR)$(mandir)/mann
@echo "Installing documentation in $(DESTDIR)$(mandir)"
@list='$(srcdir)/doc/*.n'; for i in $$list; do \
echo "Installing $$i"; \
$(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/mann" ; \
rm -f $(DESTDIR)$(mandir)/mann/`basename $$i`; \
$(INSTALL_DATA) $$i $(DESTDIR)$(mandir)/mann ; \
done
test: binaries libraries
@@ -253,21 +244,7 @@ shell: binaries libraries
@$(TCLSH) $(SCRIPT)
gdb:
$(TCLSH_ENV) $(PKG_ENV) $(GDB) $(TCLSH_PROG) $(SCRIPT)
gdb-test: binaries libraries
$(TCLSH_ENV) $(PKG_ENV) $(GDB) \
--args $(TCLSH_PROG) `@CYGPATH@ $(srcdir)/tests/all.tcl` \
$(TESTFLAGS) -singleproc 1 \
-load "package ifneeded $(PACKAGE_NAME) $(PACKAGE_VERSION) \
[list load `@CYGPATH@ $(PKG_LIB_FILE)` [string totitle $(PACKAGE_NAME)]]"
valgrind: binaries libraries
$(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) \
`@CYGPATH@ $(srcdir)/tests/all.tcl` $(TESTFLAGS)
valgrindshell: binaries libraries
$(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) $(SCRIPT)
$(TCLSH_ENV) gdb $(TCLSH_PROG) $(SCRIPT)
depend:
@@ -306,70 +283,49 @@ $(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS)
# As necessary, add $(srcdir):$(srcdir)/compat:....
#========================================================================
VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macosx
VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win
.c.@OBJEXT@:
$(COMPILE) -c `@CYGPATH@ $<` -o $@
tclsample.@OBJEXT@: sampleUuid.h
$(srcdir)/manifest.uuid:
printf "git-" >$(srcdir)/manifest.uuid
(cd $(srcdir); git rev-parse HEAD >>$(srcdir)/manifest.uuid || \
(printf "svn-r" >$(srcdir)/manifest.uuid ; \
svn info --show-item last-changed-revision >>$(srcdir)/manifest.uuid) || \
printf "unknown" >$(srcdir)/manifest.uuid)
sampleUuid.h: $(srcdir)/manifest.uuid
echo "#define SAMPLE_VERSION_UUID \\" >$@
cat $(srcdir)/manifest.uuid >>$@
echo "" >>$@
#========================================================================
# Distribution creation
# You may need to tweak this target to make it work correctly.
#========================================================================
#COMPRESS = tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar
COMPRESS = tar zcvf $(PKG_DIR).tar.gz $(PKG_DIR)
COMPRESS = gtar zcvf $(PKG_DIR).tar.gz $(PKG_DIR)
DIST_ROOT = /tmp/dist
DIST_DIR = $(DIST_ROOT)/$(PKG_DIR)
DIST_INSTALL_DATA = CPPROG='cp -p' $(INSTALL) -m 644
DIST_INSTALL_SCRIPT = CPPROG='cp -p' $(INSTALL) -m 755
dist-clean:
rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.*
dist: dist-clean $(srcdir)/manifest.uuid
$(INSTALL_DATA_DIR) $(DIST_DIR)
dist: dist-clean
mkdir -p $(DIST_DIR)
cp -p $(srcdir)/README* $(srcdir)/license* \
$(srcdir)/aclocal.m4 $(srcdir)/configure $(srcdir)/*.in \
$(DIST_DIR)/
chmod 664 $(DIST_DIR)/Makefile.in $(DIST_DIR)/aclocal.m4
chmod 775 $(DIST_DIR)/configure $(DIST_DIR)/configure.in
# TEA files
$(DIST_INSTALL_DATA) $(srcdir)/Makefile.in \
$(srcdir)/aclocal.m4 $(srcdir)/configure.ac \
$(DIST_DIR)/
$(DIST_INSTALL_SCRIPT) $(srcdir)/configure $(DIST_DIR)/
for i in $(srcdir)/*.[ch]; do \
if [ -f $$i ]; then \
cp -p $$i $(DIST_DIR)/ ; \
fi; \
done;
$(INSTALL_DATA_DIR) $(DIST_DIR)/tclconfig
$(DIST_INSTALL_DATA) $(srcdir)/tclconfig/README.txt \
$(srcdir)/manifest.uuid \
$(srcdir)/tclconfig/tcl.m4 $(srcdir)/tclconfig/install-sh \
$(DIST_DIR)/tclconfig/
mkdir $(DIST_DIR)/tclconfig
cp $(srcdir)/tclconfig/install-sh $(srcdir)/tclconfig/tcl.m4 \
$(DIST_DIR)/tclconfig/
chmod 664 $(DIST_DIR)/tclconfig/tcl.m4
chmod +x $(DIST_DIR)/tclconfig/install-sh
# Extension files
$(DIST_INSTALL_DATA) \
$(srcdir)/ChangeLog \
$(srcdir)/README.sha \
$(srcdir)/license.terms \
$(srcdir)/README \
$(srcdir)/pkgIndex.tcl.in \
$(DIST_DIR)/
list='demos doc generic library macosx tests unix win'; \
list='demos doc generic library mac tests unix win'; \
for p in $$list; do \
if test -d $(srcdir)/$$p ; then \
$(INSTALL_DATA_DIR) $(DIST_DIR)/$$p; \
$(DIST_INSTALL_DATA) $(srcdir)/$$p/* $(DIST_DIR)/$$p/; \
mkdir $(DIST_DIR)/$$p; \
cp -p $(srcdir)/$$p/*.* $(DIST_DIR)/$$p/; \
fi; \
done
@@ -381,10 +337,10 @@ dist: dist-clean $(srcdir)/manifest.uuid
#========================================================================
# Don't modify the file to clean here. Instead, set the "CLEANFILES"
# variable in configure.ac
# variable in configure.in
#========================================================================
clean:
clean:
-test -z "$(BINARIES)" || rm -f $(BINARIES)
-rm -f *.$(OBJEXT) core *.core
-test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
@@ -392,7 +348,7 @@ clean:
distclean: clean
-rm -f *.tab.c
-rm -f $(CONFIG_CLEAN_FILES)
-rm -f config.cache config.log config.status
-rm -f config.h config.cache config.log config.status
#========================================================================
# Install binary object libraries. On Windows this includes both .dll and
@@ -406,17 +362,25 @@ distclean: clean
#========================================================================
install-lib-binaries: binaries
@$(INSTALL_DATA_DIR) "$(DESTDIR)$(pkglibdir)"
@mkdir -p $(DESTDIR)$(pkglibdir)
@list='$(lib_BINARIES)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_LIBRARY) $$p $(DESTDIR)$(pkglibdir)/$$p"; \
$(INSTALL_LIBRARY) $$p "$(DESTDIR)$(pkglibdir)/$$p"; \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p; \
stub=`echo $$p|sed -e "s/.*\(stub\).*/\1/"`; \
if test "x$$stub" = "xstub"; then \
echo " $(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p"; \
$(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p; \
else \
echo " $(RANLIB) $(DESTDIR)$(pkglibdir)/$$p"; \
$(RANLIB) $(DESTDIR)$(pkglibdir)/$$p; \
fi; \
ext=`echo $$p|sed -e "s/.*\.//"`; \
if test "x$$ext" = "xdll"; then \
lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \
if test -f $$lib; then \
echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \
$(INSTALL_DATA) $$lib "$(DESTDIR)$(pkglibdir)/$$lib"; \
$(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib; \
fi; \
fi; \
fi; \
@@ -425,12 +389,12 @@ install-lib-binaries: binaries
if test -f $(srcdir)/$$p; then \
destp=`basename $$p`; \
echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \
$(INSTALL_DATA) $(srcdir)/$$p "$(DESTDIR)$(pkglibdir)/$$destp"; \
$(INSTALL_DATA) $(srcdir)/$$p $(DESTDIR)$(pkglibdir)/$$destp; \
fi; \
done
@if test "x$(SHARED_BUILD)" = "x1"; then \
echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \
$(INSTALL_DATA) pkgIndex.tcl "$(DESTDIR)$(pkglibdir)"; \
$(INSTALL_DATA) pkgIndex.tcl $(DESTDIR)$(pkglibdir); \
fi
#========================================================================
@@ -443,32 +407,33 @@ install-lib-binaries: binaries
#========================================================================
install-bin-binaries: binaries
@$(INSTALL_DATA_DIR) "$(DESTDIR)$(bindir)"
@mkdir -p $(DESTDIR)$(bindir)
@list='$(bin_BINARIES)'; for p in $$list; do \
if test -f $$p; then \
echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \
$(INSTALL_PROGRAM) $$p "$(DESTDIR)$(bindir)/$$p"; \
$(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p; \
fi; \
done
.SUFFIXES: .c .$(OBJEXT)
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
uninstall-binaries:
list='$(lib_BINARIES)'; for p in $$list; do \
rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \
rm -f $(DESTDIR)$(pkglibdir)/$$p; \
done
list='$(PKG_TCL_SOURCES)'; for p in $$list; do \
p=`basename $$p`; \
rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \
rm -f $(DESTDIR)$(pkglibdir)/$$p; \
done
list='$(bin_BINARIES)'; for p in $$list; do \
rm -f "$(DESTDIR)$(bindir)/$$p"; \
rm -f $(DESTDIR)$(bindir)/$$p; \
done
.PHONY: all binaries clean depend distclean doc install libraries test
.PHONY: gdb gdb-test valgrind valgrindshell
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
+34 -60
View File
@@ -2,9 +2,11 @@
dnl This file is an input file used by the GNU "autoconf" program to
dnl generate the file "configure", which is run during Tcl installation
dnl to configure the system for the local environment.
#
# RCS: @(#) $Id: configure.in,v 1.43 2005/07/26 19:17:05 mdejong Exp $
#-----------------------------------------------------------------------
# Sample configure.ac for Tcl Extensions. The only places you should
# Sample configure.in for Tcl Extensions. The only places you should
# need to modify this file are marked by the string __CHANGE__
#-----------------------------------------------------------------------
@@ -15,11 +17,9 @@ dnl to configure the system for the local environment.
# This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION
# set as provided. These will also be added as -D defs in your Makefile
# so you can encode the package version directly into the source files.
# This will also define a special symbol for Windows (BUILD_<PACKAGE_NAME>
# so that we create the export library with the dll.
#-----------------------------------------------------------------------
AC_INIT([sqlite],[3.47.0])
AC_INIT([sqlite], [3.32.0])
#--------------------------------------------------------------------
# Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -27,7 +27,7 @@ AC_INIT([sqlite],[3.47.0])
# as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE.
#--------------------------------------------------------------------
TEA_INIT()
TEA_INIT([3.9])
AC_CONFIG_AUX_DIR(tclconfig)
@@ -55,8 +55,8 @@ TEA_PREFIX
#-----------------------------------------------------------------------
# Standard compiler checks.
# This sets up CC by using the CC env var, or looks for gcc otherwise.
# This also calls AC_PROG_CC and a few others to create the basic setup
# necessary to compile executables.
# This also calls AC_PROG_CC, AC_PROG_INSTALL and a few others to create
# the basic setup necessary to compile executables.
#-----------------------------------------------------------------------
TEA_SETUP_COMPILER
@@ -73,19 +73,11 @@ TEA_SETUP_COMPILER
TEA_ADD_SOURCES([tclsqlite3.c])
TEA_ADD_HEADERS([])
TEA_ADD_INCLUDES([])
TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"])
TEA_ADD_LIBS([])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS4=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS5=1])
TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_GEOPOLY=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_MATH_FUNCTIONS=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DESERIALIZE=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBPAGE_VTAB=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_BYTECODE_VTAB=1])
TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBSTAT_VTAB=1])
TEA_ADD_STUB_SOURCES([])
TEA_ADD_TCL_SOURCES([])
@@ -107,31 +99,6 @@ if test x$with_system_sqlite != xno; then
LIBS="$LIBS -lsqlite3"])])
fi
#--------------------------------------------------------------------
# __CHANGE__
#
# You can add more files to clean if your extension creates any extra
# files by extending CLEANFILES.
# Add pkgIndex.tcl if it is generated in the Makefile instead of ./configure
# and change Makefile.in to move it from CONFIG_CLEAN_FILES to BINARIES var.
#
# A few miscellaneous platform-specific items:
# TEA_ADD_* any platform specific compiler/build info here.
#--------------------------------------------------------------------
#CLEANFILES="$CLEANFILES pkgIndex.tcl"
if test "${TEA_PLATFORM}" = "windows" ; then
# Ensure no empty if clauses
:
#TEA_ADD_SOURCES([win/winFile.c])
#TEA_ADD_INCLUDES([-I\"$(${CYGPATH} ${srcdir}/win)\"])
else
# Ensure no empty else clauses
:
#TEA_ADD_SOURCES([unix/unixFile.c])
#TEA_ADD_LIBS([-lsuperfly])
fi
#--------------------------------------------------------------------
# __CHANGE__
# Choose which headers you need. Extension authors should try very
@@ -184,6 +151,28 @@ TEA_CONFIG_CFLAGS
TEA_ENABLE_SYMBOLS
#--------------------------------------------------------------------
# Everyone should be linking against the Tcl stub library. If you
# can't for some reason, remove this definition. If you aren't using
# stubs, you also need to modify the SHLIB_LD_LIBS setting below to
# link against the non-stubbed Tcl library. Add Tk too if necessary.
#--------------------------------------------------------------------
AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs])
#AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
#--------------------------------------------------------------------
# Redefine fdatasync as fsync on systems that lack fdatasync
#--------------------------------------------------------------------
#
#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync))
# Check for library functions that SQLite can optionally use.
AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
AC_FUNC_STRERROR_R
#--------------------------------------------------------------------
# This macro generates a line to use when building a library. It
# depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS,
@@ -204,24 +193,9 @@ TEA_PROG_TCLSH
#TEA_PROG_WISH
#--------------------------------------------------------------------
# Setup a *Config.sh.in configuration file.
# Finally, substitute all of the various values into the Makefile.
# You may alternatively have a special pkgIndex.tcl.in or other files
# which require substituting th AC variables in. Include these here.
#--------------------------------------------------------------------
#TEA_EXPORT_CONFIG([sample])
#AC_SUBST(SAMPLE_VAR)
#--------------------------------------------------------------------
# Specify files to substitute AC variables in. You may alternatively
# have a special pkgIndex.tcl.in or other files which require
# substituting the AC variables in. Include these here.
#--------------------------------------------------------------------
AC_CONFIG_FILES([Makefile pkgIndex.tcl])
#AC_CONFIG_FILES([sampleConfig.sh])
#--------------------------------------------------------------------
# Finally, substitute all of the various values into the files
# specified with AC_CONFIG_FILES.
#--------------------------------------------------------------------
AC_OUTPUT
AC_OUTPUT([Makefile pkgIndex.tcl])
+6 -9
View File
@@ -1,10 +1,7 @@
# -*- tcl -*-
# Tcl package index file, version 1.1
#
if {[package vsatisfies [package provide Tcl] 9.0-]} {
package ifneeded sqlite3 @PACKAGE_VERSION@ \
[list load [file join $dir @PKG_LIB_FILE9@] sqlite3]
} else {
package ifneeded sqlite3 @PACKAGE_VERSION@ \
[list load [file join $dir @PKG_LIB_FILE8@] sqlite3]
}
# Tcl package index file
#
# Note sqlite*3* init specifically
#
package ifneeded sqlite3 @PACKAGE_VERSION@ \
[list load [file join $dir @PKG_LIB_FILE@] Sqlite3]
File diff suppressed because it is too large Load Diff
+1 -12
View File
@@ -251,18 +251,7 @@ INCLUDES = $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \
-I"$(GENERICDIR)" -I"$(ROOT)\.."
BASE_CLFAGS = $(cflags) $(cdebug) $(crt) $(INCLUDES) \
-DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \
-DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 \
-DSQLITE_ENABLE_FTS4=1 \
-DSQLITE_ENABLE_FTS5=1 \
-DSQLITE_3_SUFFIX_ONLY=1 \
-DSQLITE_ENABLE_RTREE=1 \
-DSQLITE_ENABLE_GEOPOLY=1 \
-DSQLITE_ENABLE_MATH_FUNCTIONS=1 \
-DSQLITE_ENABLE_DESERIALIZE=1 \
-DSQLITE_ENABLE_DBPAGE_VTAB=1 \
-DSQLITE_ENABLE_BYTECODE_VTAB=1 \
-DSQLITE_ENABLE_DBSTAT_VTAB=1
-DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1
CON_CFLAGS = $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1
TCL_CFLAGS = -DBUILD_sqlite -DUSE_TCL_STUBS \
-DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \
+1
View File
@@ -708,3 +708,4 @@ TK_INCLUDES = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib"
!message *** Link options '$(LINKERFLAGS)'
!endif
+7 -18
View File
@@ -1,4 +1,4 @@
/* sqlite_cfg.h.in. Generated from configure.ac by autoheader. */
/* config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if you have the <dlfcn.h> header file. */
#undef HAVE_DLFCN_H
@@ -45,16 +45,16 @@
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the `pread' function. */
/* Define to 1 if you have the pread() function. */
#undef HAVE_PREAD
/* Define to 1 if you have the `pread64' function. */
/* Define to 1 if you have the pread64() function. */
#undef HAVE_PREAD64
/* Define to 1 if you have the `pwrite' function. */
/* Define to 1 if you have the pwrite() function. */
#undef HAVE_PWRITE
/* Define to 1 if you have the `pwrite64' function. */
/* Define to 1 if you have the pwrite64() function. */
#undef HAVE_PWRITE64
/* Define to 1 if you have the <stdint.h> header file. */
@@ -63,7 +63,7 @@
/* Define to 1 if you have the <stdlib.h> header file. */
#undef HAVE_STDLIB_H
/* Define to 1 if you have the `strchrnul' function. */
/* Define to 1 if you have the strchrnul() function */
#undef HAVE_STRCHRNUL
/* Define to 1 if you have the <strings.h> header file. */
@@ -99,12 +99,9 @@
/* Define to 1 if you have the `usleep' function. */
#undef HAVE_USLEEP
/* Define to 1 if you have the `utime' function. */
/* Define to 1 if you have the utime() library function. */
#undef HAVE_UTIME
/* Define to 1 if you have the <zlib.h> header file. */
#undef HAVE_ZLIB_H
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
#undef LT_OBJDIR
@@ -121,20 +118,12 @@
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the home page for this package. */
#undef PACKAGE_URL
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* Define to 1 if you have the ANSI C header files. */
#undef STDC_HEADERS
/* Enable large inode numbers on Mac OS X 10.5. */
#ifndef _DARWIN_USE_64_BIT_INODE
# define _DARWIN_USE_64_BIT_INODE 1
#endif
/* Number of bits in a file offset, on hosts where this is settable. */
#undef _FILE_OFFSET_BITS
Vendored
+299 -295
View File
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
# Generated by GNU Autoconf 2.69 for sqlite 3.37.0.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='sqlite'
PACKAGE_TARNAME='sqlite'
PACKAGE_VERSION='3.47.0'
PACKAGE_STRING='sqlite 3.47.0'
PACKAGE_VERSION='3.37.0'
PACKAGE_STRING='sqlite 3.37.0'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@@ -770,17 +770,28 @@ ac_includes_default="\
ac_subst_vars='LTLIBOBJS
LIBOBJS
BUILD_CFLAGS
AMALGAMATION_LINE_MACROS
USE_GCOV
OPT_FEATURE_FLAGS
HAVE_ZLIB
USE_AMALGAMATION
AMALGAMATION_LINE_MACROS
TARGET_DEBUG
TARGET_HAVE_LINENOISE
TARGET_HAVE_EDITLINE
TARGET_HAVE_READLINE
TARGET_READLINE_INC
TARGET_READLINE_LIBS
HAVE_TCL
TCL_SHLIB_SUFFIX
TCL_STUB_LIB_SPEC
TCL_STUB_LIB_FLAG
TCL_STUB_LIB_FILE
TCL_LIB_SPEC
TCL_LIB_FLAG
TCL_LIB_FILE
TCL_INCLUDE_SPEC
TCL_SRC_DIR
TCL_BIN_DIR
TCL_VERSION
TARGET_EXEEXT
SQLITE_OS_WIN
SQLITE_OS_UNIX
@@ -789,16 +800,10 @@ TEMP_STORE
ALLOWRELEASE
SQLITE_THREADSAFE
BUILD_CC
HAVE_WASI_SDK
RELEASE
VERSION
program_prefix
TSTRNNR_OPTS
TCLLIBDIR
HAVE_TCL
TCL_STUB_LIB_SPEC
TCL_LIB_SPEC
TCL_INCLUDE_SPEC
TCLSH_CMD
INSTALL_DATA
INSTALL_SCRIPT
@@ -887,30 +892,27 @@ enable_fast_install
with_gnu_ld
enable_libtool_lock
enable_largefile
with_tclsh
with_tcl
enable_tcl
enable_test_status
with_wasi_sdk
enable_threadsafe
enable_releasemode
enable_tempstore
enable_tcl
with_tcl
enable_editline
enable_readline
with_readline_lib
with_readline_inc
with_linenoise
enable_debug
enable_amalgamation
amalgamation_line_macros
enable_load_extension
enable_math
enable_json
enable_all
enable_memsys5
enable_memsys3
enable_fts3
enable_fts4
enable_fts5
enable_json1
enable_update_limit
enable_geopoly
enable_rtree
@@ -1467,7 +1469,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.
\`configure' configures sqlite 3.37.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1532,7 +1534,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of sqlite 3.47.0:";;
short | recursive ) echo "Configuration of sqlite 3.37.0:";;
esac
cat <<\_ACEOF
@@ -1546,13 +1548,11 @@ Optional Features:
optimize for fast installation [default=yes]
--disable-libtool-lock avoid locking (might break parallel builds)
--disable-largefile omit support for large files
--disable-tcl omit building accessory programs that require
TCL-dev
--enable-test-status Full-screen status of tests
--disable-threadsafe Disable mutexing
--enable-releasemode Support libtool link to release mode
--enable-tempstore Use an in-ram database for temporary tables
(never,no,yes,always)
--disable-tcl do not build TCL extension
--enable-editline enable BSD editline support
--disable-readline disable readline support
--enable-debug enable debugging & verbose explain
@@ -1561,13 +1561,13 @@ Optional Features:
--disable-load-extension
Disable loading of external extensions
--disable-math Disable math functions
--disable-json Disable JSON functions
--enable-all Enable FTS4, FTS5, Geopoly, RTree, Sessions
--enable-all Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions
--enable-memsys5 Enable MEMSYS5
--enable-memsys3 Enable MEMSYS3
--enable-fts3 Enable the FTS3 extension
--enable-fts4 Enable the FTS4 extension
--enable-fts5 Enable the FTS5 extension
--enable-json1 Enable the JSON1 extension
--enable-update-limit Enable the UPDATE/DELETE LIMIT clause
--enable-geopoly Enable the GEOPOLY extension
--enable-rtree Enable the RTREE extension
@@ -1580,13 +1580,10 @@ Optional Packages:
--with-pic try to use only PIC/non-PIC objects [default=use
both]
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-tclsh=PATHNAME full pathname of a tclsh to use
--with-tcl=DIR directory containing (tclConfig.sh)
--with-wasi-sdk=DIR directory containing the WASI SDK. Triggers
cross-compile to WASM.
--with-tcl=DIR directory containing tcl configuration
(tclConfig.sh)
--with-readline-lib specify readline library
--with-readline-inc specify readline include paths
--with-linenoise=DIR source directory for linenoise library
Some influential environment variables:
CC C compiler command
@@ -1665,7 +1662,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
sqlite configure 3.47.0
sqlite configure 3.37.0
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2084,7 +2081,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by sqlite $as_me 3.47.0, which was
It was created by sqlite $as_me 3.37.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@@ -3942,13 +3939,13 @@ if ${lt_cv_nm_interface+:} false; then :
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
(eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
(eval echo "\"\$as_me:3940: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
(eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval echo "\"\$as_me:3943: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
(eval echo "\"\$as_me:3951: output\"" >&5)
(eval echo "\"\$as_me:3946: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@@ -5154,7 +5151,7 @@ ia64-*-hpux*)
;;
*-*-irix6*)
# Find out which ABI we are using.
echo '#line 5157 "configure"' > conftest.$ac_ext
echo '#line 5152 "configure"' > conftest.$ac_ext
if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
(eval $ac_compile) 2>&5
ac_status=$?
@@ -6679,11 +6676,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
(eval echo "\"\$as_me:6677: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:6686: \$? = $ac_status" >&5
echo "$as_me:6681: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -7018,11 +7015,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7016: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
echo "$as_me:7025: \$? = $ac_status" >&5
echo "$as_me:7020: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -7123,11 +7120,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7121: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7130: \$? = $ac_status" >&5
echo "$as_me:7125: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -7178,11 +7175,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
(eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
(eval echo "\"\$as_me:7176: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
echo "$as_me:7185: \$? = $ac_status" >&5
echo "$as_me:7180: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -9558,7 +9555,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9561 "configure"
#line 9556 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -9654,7 +9651,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 9657 "configure"
#line 9652 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -10311,31 +10308,18 @@ done
USE_AMALGAMATION=1
#########
# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
# By default, amalgamation sqlite3.c will have #line directives.
# This is a build option not shown by ./configure --help
# To control it, use configure option: amalgamation_line_macros=?
# where ? is no to suppress #line directives or yes to create them.
AMALGAMATION_LINE_MACROS=--linemacros=0
#########
# See whether we can run specific tclsh versions known to work well;
# if not, then we fall back to plain tclsh.
# TODO: try other versions before falling back?
#
# Check whether --with-tclsh was given.
if test "${with_tclsh+set}" = set; then :
withval=$with_tclsh;
fi
# Check whether --with-tcl was given.
if test "${with_tcl+set}" = set; then :
withval=$with_tcl;
fi
# Check whether --enable-tcl was given.
if test "${enable_tcl+set}" = set; then :
enableval=$enable_tcl; use_tcl=$enableval
else
use_tcl=yes
fi
original_use_tcl=${use_tcl}
if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
for ac_prog in tclsh8.6 tclsh tclsh9.0
for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
@@ -10378,99 +10362,10 @@ fi
done
test -n "$TCLSH_CMD" || TCLSH_CMD="none"
with_tclsh=${TCLSH_CMD}
fi
if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
TCLSH_CMD=${with_tclsh}
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5
$as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; }
if test x"${use_tcl}" = "xyes"; then
with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
if test x"${with_tcl}" != x; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5
$as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5
$as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;}
use_tcl=no
fi
fi
fi
if test x"${use_tcl}" = "xyes"; then
if test x"${with_tcl}" != x; then
if test -r ${with_tcl}/tclConfig.sh; then
tclconfig="${with_tcl}/tclConfig.sh"
else
for i in tcl8.6 tcl9.0 lib; do
if test -r ${with_tcl}/$i/tclConfig.sh; then
tclconfig=${with_tcl}/$i/tclConfig.sh
break
fi
done
fi
if test ! -r "${tclconfig}"; then
as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5
fi
else
# If we have not yet found a tclConfig.sh file, look in $libdir whic is
# set automatically by autoconf or by the --prefix command-line option.
# See https://sqlite.org/forum/forumpost/e04e693439a22457
libdir=${prefix}/lib
if test -r ${libdir}/tclConfig.sh; then
tclconfig=${libdir}/tclConfig.sh
else
for i in tcl8.6 tcl9.0 lib; do
if test -r ${libdir}/$i/tclConfig.sh; then
tclconfig=${libdir}/$i/tclConfig.sh
break
fi
done
fi
if test ! -r "${tclconfig}"; then
as_fn_error $? "cannot find a usable tclConfig.sh file.
Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
SQLite does not use TCL internally, but TCL is required to build SQLite
from canonical sources and TCL is required for testing." "$LINENO" 5
fi
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5
$as_echo "loading TCL configuration from ${tclconfig}" >&6; }
. ${tclconfig}
# There are lots of other configuration variables that are provided by the
# tclConfig.sh file and that could be included here. But as of right now,
# TCL_LIB_SPEC is the only what that the Makefile uses.
HAVE_TCL=1
elif test x"${original_use_tcl}" = "xno"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5
$as_echo "unable to run tests because of --disable-tcl" >&6; }
HAVE_TCL=0
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5
$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; }
HAVE_TCL=0
fi
if test x"$TCLSH_CMD" == x; then
TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
if test ! -x ${TCLSH_CMD}; then
TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
if test ! -x ${TCLSH_CMD_2}; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&5
$as_echo "$as_me: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&2;}
TCLSH_CMD=none
else
TCLSH_CMD=${TCLSH_CMD_2}
fi
fi
fi
if test "$TCLSH_CMD" = "none"; then
# If we can't find a local tclsh, then building the amalgamation will fail.
# We act as though --disable-amalgamation has been used.
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&5
$as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;}
echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
USE_AMALGAMATION=0
TCLSH_CMD="tclsh"
fi
@@ -10478,6 +10373,7 @@ fi
if test "x${TCLLIBDIR+set}" != "xset" ; then
TCLLIBDIR='$(libdir)'
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
if test -d $i ; then
TCLLIBDIR=$i
@@ -10487,23 +10383,6 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
TCLLIBDIR="${TCLLIBDIR}/sqlite3"
fi
#########
# Set up options for running tests.
#
# Check whether --enable-test-status was given.
if test "${enable_test_status+set}" = set; then :
enableval=$enable_test_status; use_vt100=$enableval
else
use_vt100=no
fi
if test $use_vt100 != no; then
TSTRNNR_OPTS=--status
else
TSTRNNR_OPTS=
fi
#########
# Set up an appropriate program prefix
@@ -10522,68 +10401,6 @@ RELEASE=`cat $srcdir/VERSION`
$as_echo "$as_me: Release set to $RELEASE" >&6;}
##########
# Handle --with-wasi-sdk=DIR
#
# This must be early because it changes the toolchain.
#
# Check whether --with-wasi-sdk was given.
if test "${with_wasi_sdk+set}" = set; then :
withval=$with_wasi_sdk; with_wasisdk=${withval}
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WASI SDK directory" >&5
$as_echo_n "checking for WASI SDK directory... " >&6; }
if ${ac_cv_c_wasi_sdk+:} false; then :
$as_echo_n "(cached) " >&6
else
# First check to see if --with-tcl was specified.
if test x"${with_wasi_sdk}" != x ; then
if ! test -d "${with_wasi_sdk}" ; then
as_fn_error $? "${with_wasi_sdk} directory doesn't exist" "$LINENO" 5
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&5
$as_echo "${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&6; }
use_wasi_sdk=yes
else
use_wasi_sdk=no
fi
fi
if test "${use_wasi_sdk}" = "no" ; then
HAVE_WASI_SDK=""
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
else
HAVE_WASI_SDK=1
# Changing --host and --target have no effect here except to possibly
# cause confusion. autoconf has finished processing them by this
# point.
#
# host_alias=wasm32-wasi
# target=wasm32-wasi
#
# Merely changing CC and LD to the wasi-sdk's is enough to get
# sqlite3.o building in WASM format.
CC="${with_wasi_sdk}/bin/clang"
LD="${with_wasi_sdk}/bin/wasm-ld"
RANLIB="${with_wasi_sdk}/bin/llvm-ranlib"
cross_compiling=yes
enable_threadsafe=no
use_tcl=no
enable_tcl=no
# libtool is apparently hard-coded to use gcc for linking DLLs, so
# we disable the DLL build...
enable_shared=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
#########
# Locate a compiler for the build machine. This compiler should
# generate command-line programs that run on the build machine.
@@ -10891,6 +10708,213 @@ fi
##########
# Figure out all the parameters needed to compile against Tcl.
#
# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
# macros in the in the tcl.m4 file of the standard TCL distribution.
# Those macros could not be used directly since we have to make some
# minor changes to accomodate systems that do not have TCL installed.
#
# Check whether --enable-tcl was given.
if test "${enable_tcl+set}" = set; then :
enableval=$enable_tcl; use_tcl=$enableval
else
use_tcl=yes
fi
if test "${use_tcl}" = "yes" ; then
# Check whether --with-tcl was given.
if test "${with_tcl+set}" = set; then :
withval=$with_tcl; with_tclconfig=${withval}
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5
$as_echo_n "checking for Tcl configuration... " >&6; }
if ${ac_cv_c_tclconfig+:} false; then :
$as_echo_n "(cached) " >&6
else
# First check to see if --with-tcl was specified.
if test x"${with_tclconfig}" != x ; then
if test -f "${with_tclconfig}/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
else
as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5
fi
fi
# Start autosearch by asking tclsh
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# On ubuntu 14.10, $auto_path on tclsh is not quite correct.
# So try again after applying corrections.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# Recent versions of Xcode on Macs hid the tclConfig.sh file
# in a strange place.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
../tcl \
`ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
`ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
`ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \
../../tcl \
`ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
`ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \
`ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \
../../../tcl \
`ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
`ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \
`ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null`
do
if test -f "$i/unix/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
break
fi
done
fi
# check in a few common install locations
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
`ls -d ${libdir} 2>/dev/null` \
`ls -d /usr/local/lib 2>/dev/null` \
`ls -d /usr/contrib/lib 2>/dev/null` \
`ls -d /usr/lib 2>/dev/null`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i; pwd)`
break
fi
done
fi
# check in a few other private locations
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
${srcdir}/../tcl \
`ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
`ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \
`ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null`
do
if test -f "$i/unix/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
break
fi
done
fi
fi
if test x"${ac_cv_c_tclconfig}" = x ; then
use_tcl=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5
$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5
$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5
$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;}
else
TCL_BIN_DIR=${ac_cv_c_tclconfig}
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5
$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5
$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; }
if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5
$as_echo "loading" >&6; }
. $TCL_BIN_DIR/tclConfig.sh
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5
$as_echo "file not found" >&6; }
fi
#
# If the TCL_BIN_DIR is the build directory (not the install directory),
# then set the common variable name to the value of the build variables.
# For example, the variable TCL_LIB_SPEC will be set to the value
# of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
# instead of TCL_BUILD_LIB_SPEC since it will work with both an
# installed and uninstalled version of Tcl.
#
if test -f $TCL_BIN_DIR/Makefile ; then
TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
fi
#
# eval is required to do the TCL_DBGX substitution
#
eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
fi
fi
if test "${use_tcl}" = "no" ; then
HAVE_TCL=""
else
HAVE_TCL=1
fi
##########
# Figure out what C libraries are required to compile programs
# that use "readline()" library.
@@ -11163,27 +11187,6 @@ fi
fi
fi
# Check whether --with-linenoise was given.
if test "${with_linenoise+set}" = set; then :
withval=$with_linenoise; with_linenoise=$withval
else
with_linenoise="no"
fi
if test "x$with_linenoise" != "xno"; then
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
TARGET_HAVE_LINENOISE=1
TARGET_READLINE_INC="-I${with_linenoise}"
TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
echo "using linenoise source code at ${with_linenoise}"
else
TARGET_HAVE_LINENOISE=0
echo "not using linenoise"
fi
@@ -11260,7 +11263,7 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5
$as_echo_n "checking build type... " >&6; }
if test "${enable_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5
$as_echo "debug" >&6; }
else
@@ -11272,7 +11275,6 @@ fi
#########
# See whether we should use the amalgamation to build
# Check whether --enable-amalgamation was given.
if test "${enable_amalgamation+set}" = set; then :
enableval=$enable_amalgamation;
@@ -11282,6 +11284,19 @@ if test "${enable_amalgamation}" = "no" ; then
USE_AMALGAMATION=0
fi
########
# See whether --disable
if test "${amalgamation_line_macros+set}" = set; then :
enableval=$amalgamation_line_macros;
fi
if test "${amalgamation_line_macros}" = "yes" ; then
AMALGAMATION_LINE_MACROS=--linemacros=1
fi
if test "${amalgamation_line_macros}" = "no" ; then
AMALGAMATION_LINE_MACROS=--linemacros=0
fi
#########
# Look for zlib. Only needed by extensions and by the sqlite3.exe shell
@@ -11503,24 +11518,6 @@ fi
fi
##########
# Do we want to support JSON functions
#
# Check whether --enable-json was given.
if test "${enable_json+set}" = set; then :
enableval=$enable_json;
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support JSON functions" >&5
$as_echo_n "checking whether to support JSON functions... " >&6; }
if test "$enable_json" = "no"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON"
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
########
# The --enable-all argument is short-hand to enable
@@ -11725,6 +11722,24 @@ else
$as_echo "no" >&6; }
fi
#########
# See whether we should enable JSON1
# Check whether --enable-json1 was given.
if test "${enable_json1+set}" = set; then :
enableval=$enable_json1;
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support JSON" >&5
$as_echo_n "checking whether to support JSON... " >&6; }
if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
#########
# See whether we should enable the LIMIT clause on UPDATE and DELETE
# statements.
@@ -11867,21 +11882,10 @@ else
fi
#########
# Enable/disabled amalagamation line macros
########
AMALGAMATION_LINE_MACROS=--linemacros=0
if test "${amalgamation_line_macros}" = "yes" ; then
AMALGAMATION_LINE_MACROS=--linemacros=1
fi
if test "${amalgamation_line_macros}" = "no" ; then
AMALGAMATION_LINE_MACROS=--linemacros=0
fi
#########
# Output the config header
ac_config_headers="$ac_config_headers sqlite_cfg.h"
ac_config_headers="$ac_config_headers config.h"
#########
@@ -12396,7 +12400,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by sqlite $as_me 3.47.0, which was
This file was extended by sqlite $as_me 3.37.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -12462,7 +12466,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
sqlite config.status 3.47.0
sqlite config.status 3.37.0
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
@@ -12844,7 +12848,7 @@ for ac_config_target in $ac_config_targets
do
case $ac_config_target in
"libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;;
"sqlite_cfg.h") CONFIG_HEADERS="$CONFIG_HEADERS sqlite_cfg.h" ;;
"config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
"Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
"sqlite3.pc") CONFIG_FILES="$CONFIG_FILES sqlite3.pc" ;;
+232 -234
View File
@@ -70,11 +70,11 @@
# target platform. "" for Unix and ".exe" for windows.
#
# This configure.in file is easy to reuse on other projects. Just
# change the argument to AC_INIT. And disable any features that
# change the argument to AC_INIT(). And disable any features that
# you don't need (for example BLT) by erasing or commenting out
# the corresponding code.
#
AC_INIT([sqlite],m4_esyscmd(cat VERSION | tr -d '\n'))
AC_INIT(sqlite, m4_esyscmd([cat VERSION | tr -d '\n']))
dnl Make sure the local VERSION file matches this configure script
sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@@ -88,7 +88,7 @@ fi
#########
# Programs needed
#
LT_INIT
AC_PROG_LIBTOOL
AC_PROG_INSTALL
#########
@@ -116,101 +116,15 @@ AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_s
USE_AMALGAMATION=1
#########
# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
#
AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use]))
AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)]))
AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]),
[use_tcl=$enableval],[use_tcl=yes])
original_use_tcl=${use_tcl}
if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none)
with_tclsh=${TCLSH_CMD}
fi
if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
TCLSH_CMD=${with_tclsh}
AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"])
if test x"${use_tcl}" = "xyes"; then
with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
if test x"${with_tcl}" != x; then
AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}])
else
AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh])
use_tcl=no
fi
fi
fi
if test x"${use_tcl}" = "xyes"; then
if test x"${with_tcl}" != x; then
if test -r ${with_tcl}/tclConfig.sh; then
tclconfig="${with_tcl}/tclConfig.sh"
else
for i in tcl8.6 tcl9.0 lib; do
if test -r ${with_tcl}/$i/tclConfig.sh; then
tclconfig=${with_tcl}/$i/tclConfig.sh
break
fi
done
fi
if test ! -r "${tclconfig}"; then
AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}])
fi
else
# If we have not yet found a tclConfig.sh file, look in $libdir whic is
# set automatically by autoconf or by the --prefix command-line option.
# See https://sqlite.org/forum/forumpost/e04e693439a22457
libdir=${prefix}/lib
if test -r ${libdir}/tclConfig.sh; then
tclconfig=${libdir}/tclConfig.sh
else
for i in tcl8.6 tcl9.0 lib; do
if test -r ${libdir}/$i/tclConfig.sh; then
tclconfig=${libdir}/$i/tclConfig.sh
break
fi
done
fi
if test ! -r "${tclconfig}"; then
AC_MSG_ERROR([cannot find a usable tclConfig.sh file.
Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
SQLite does not use TCL internally, but TCL is required to build SQLite
from canonical sources and TCL is required for testing.])
fi
fi
AC_MSG_RESULT([loading TCL configuration from ${tclconfig}])
. ${tclconfig}
AC_SUBST(TCL_INCLUDE_SPEC)
AC_SUBST(TCL_LIB_SPEC)
AC_SUBST(TCL_STUB_LIB_SPEC)
# There are lots of other configuration variables that are provided by the
# tclConfig.sh file and that could be included here. But as of right now,
# TCL_LIB_SPEC is the only what that the Makefile uses.
HAVE_TCL=1
elif test x"${original_use_tcl}" = "xno"; then
AC_MSG_RESULT([unable to run tests because of --disable-tcl])
HAVE_TCL=0
else
AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located])
HAVE_TCL=0
fi
AC_SUBST(HAVE_TCL)
if test x"$TCLSH_CMD" == x; then
TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
if test ! -x ${TCLSH_CMD}; then
TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
if test ! -x ${TCLSH_CMD_2}; then
AC_MSG_WARN([cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}])
TCLSH_CMD=none
else
TCLSH_CMD=${TCLSH_CMD_2}
fi
fi
fi
# See whether we can run specific tclsh versions known to work well;
# if not, then we fall back to plain tclsh.
# TODO: try other versions before falling back?
#
AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
if test "$TCLSH_CMD" = "none"; then
# If we can't find a local tclsh, then building the amalgamation will fail.
# We act as though --disable-amalgamation has been used.
AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.])
echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
USE_AMALGAMATION=0
TCLSH_CMD="tclsh"
fi
@@ -218,6 +132,7 @@ AC_SUBST(TCLSH_CMD)
AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin])
if test "x${TCLLIBDIR+set}" != "xset" ; then
TCLLIBDIR='$(libdir)'
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
if test -d $i ; then
TCLLIBDIR=$i
@@ -227,18 +142,6 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
TCLLIBDIR="${TCLLIBDIR}/sqlite3"
fi
#########
# Set up options for running tests.
#
AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[Full-screen status of tests]),
[use_vt100=$enableval],[use_vt100=no])
if test $use_vt100 != no; then
TSTRNNR_OPTS=--status
else
TSTRNNR_OPTS=
fi
AC_SUBST(TSTRNNR_OPTS)
#########
# Set up an appropriate program prefix
@@ -255,56 +158,6 @@ RELEASE=`cat $srcdir/VERSION`
AC_MSG_NOTICE(Release set to $RELEASE)
AC_SUBST(RELEASE)
##########
# Handle --with-wasi-sdk=DIR
#
# This must be early because it changes the toolchain.
#
AC_ARG_WITH(wasi-sdk,
AS_HELP_STRING([--with-wasi-sdk=DIR],
[directory containing the WASI SDK. Triggers cross-compile to WASM.]), with_wasisdk=${withval})
AC_MSG_CHECKING([for WASI SDK directory])
AC_CACHE_VAL(ac_cv_c_wasi_sdk,[
# First check to see if --with-tcl was specified.
if test x"${with_wasi_sdk}" != x ; then
if ! test -d "${with_wasi_sdk}" ; then
AC_MSG_ERROR([${with_wasi_sdk} directory doesn't exist])
fi
AC_MSG_RESULT([${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL])
use_wasi_sdk=yes
else
use_wasi_sdk=no
fi
])
if test "${use_wasi_sdk}" = "no" ; then
HAVE_WASI_SDK=""
AC_MSG_RESULT([no])
else
HAVE_WASI_SDK=1
# Changing --host and --target have no effect here except to possibly
# cause confusion. autoconf has finished processing them by this
# point.
#
# host_alias=wasm32-wasi
# target=wasm32-wasi
#
# Merely changing CC and LD to the wasi-sdk's is enough to get
# sqlite3.o building in WASM format.
CC="${with_wasi_sdk}/bin/clang"
LD="${with_wasi_sdk}/bin/wasm-ld"
RANLIB="${with_wasi_sdk}/bin/llvm-ranlib"
cross_compiling=yes
enable_threadsafe=no
use_tcl=no
enable_tcl=no
# libtool is apparently hard-coded to use gcc for linking DLLs, so
# we disable the DLL build...
enable_shared=no
AC_MSG_RESULT([yes])
fi
AC_SUBST(HAVE_WASI_SDK)
#########
# Locate a compiler for the build machine. This compiler should
# generate command-line programs that run on the build machine.
@@ -326,7 +179,7 @@ AC_SUBST(BUILD_CC)
# Do we want to support multithreaded use of sqlite
#
AC_ARG_ENABLE(threadsafe,
AS_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
AC_MSG_CHECKING([whether to support threadsafe operation])
if test "$enable_threadsafe" = "no"; then
SQLITE_THREADSAFE=0
@@ -346,7 +199,7 @@ fi
# Do we want to support release
#
AC_ARG_ENABLE(releasemode,
AS_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
AC_MSG_CHECKING([whether to support shared library linked as release mode or not])
if test "$enable_releasemode" = "no"; then
ALLOWRELEASE=""
@@ -361,7 +214,7 @@ AC_SUBST(ALLOWRELEASE)
# Do we want temporary databases in memory
#
AC_ARG_ENABLE(tempstore,
AS_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
AC_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
AC_MSG_CHECKING([whether to use an in-ram database for temporary tables])
case "$enable_tempstore" in
never )
@@ -401,15 +254,7 @@ else
AC_MSG_RESULT(unknown)
fi
if test "$CYGWIN" != "yes"; then
m4_warn([obsolete],
[AC_CYGWIN is obsolete: use AC_CANONICAL_HOST and check if $host_os
matches *cygwin*])dnl
AC_CANONICAL_HOST
case $host_os in
*cygwin* ) CYGWIN=yes;;
* ) CYGWIN=no;;
esac
AC_CYGWIN
fi
if test "$CYGWIN" = "yes"; then
BUILD_EXEEXT=.exe
@@ -436,6 +281,190 @@ AC_SUBST(SQLITE_OS_UNIX)
AC_SUBST(SQLITE_OS_WIN)
AC_SUBST(TARGET_EXEEXT)
##########
# Figure out all the parameters needed to compile against Tcl.
#
# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
# macros in the in the tcl.m4 file of the standard TCL distribution.
# Those macros could not be used directly since we have to make some
# minor changes to accomodate systems that do not have TCL installed.
#
AC_ARG_ENABLE(tcl, AC_HELP_STRING([--disable-tcl],[do not build TCL extension]),
[use_tcl=$enableval],[use_tcl=yes])
if test "${use_tcl}" = "yes" ; then
AC_ARG_WITH(tcl, AC_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
AC_MSG_CHECKING([for Tcl configuration])
AC_CACHE_VAL(ac_cv_c_tclconfig,[
# First check to see if --with-tcl was specified.
if test x"${with_tclconfig}" != x ; then
if test -f "${with_tclconfig}/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
else
AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
fi
fi
# Start autosearch by asking tclsh
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# On ubuntu 14.10, $auto_path on tclsh is not quite correct.
# So try again after applying corrections.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# Recent versions of Xcode on Macs hid the tclConfig.sh file
# in a strange place.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
../tcl \
`ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
`ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
`ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
../../tcl \
`ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
`ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
`ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
../../../tcl \
`ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
`ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
`ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null`
do
if test -f "$i/unix/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
break
fi
done
fi
# check in a few common install locations
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
`ls -d ${libdir} 2>/dev/null` \
`ls -d /usr/local/lib 2>/dev/null` \
`ls -d /usr/contrib/lib 2>/dev/null` \
`ls -d /usr/lib 2>/dev/null`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i; pwd)`
break
fi
done
fi
# check in a few other private locations
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
${srcdir}/../tcl \
`ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
`ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \
`ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null`
do
if test -f "$i/unix/tclConfig.sh" ; then
ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
break
fi
done
fi
])
if test x"${ac_cv_c_tclconfig}" = x ; then
use_tcl=no
AC_MSG_WARN(Can't find Tcl configuration definitions)
AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***)
AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***)
else
TCL_BIN_DIR=${ac_cv_c_tclconfig}
AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh)
AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh])
if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
AC_MSG_RESULT([loading])
. $TCL_BIN_DIR/tclConfig.sh
else
AC_MSG_RESULT([file not found])
fi
#
# If the TCL_BIN_DIR is the build directory (not the install directory),
# then set the common variable name to the value of the build variables.
# For example, the variable TCL_LIB_SPEC will be set to the value
# of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
# instead of TCL_BUILD_LIB_SPEC since it will work with both an
# installed and uninstalled version of Tcl.
#
if test -f $TCL_BIN_DIR/Makefile ; then
TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
fi
#
# eval is required to do the TCL_DBGX substitution
#
eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
AC_SUBST(TCL_VERSION)
AC_SUBST(TCL_BIN_DIR)
AC_SUBST(TCL_SRC_DIR)
AC_SUBST(TCL_INCLUDE_SPEC)
AC_SUBST(TCL_LIB_FILE)
AC_SUBST(TCL_LIB_FLAG)
AC_SUBST(TCL_LIB_SPEC)
AC_SUBST(TCL_STUB_LIB_FILE)
AC_SUBST(TCL_STUB_LIB_FLAG)
AC_SUBST(TCL_STUB_LIB_SPEC)
AC_SUBST(TCL_SHLIB_SUFFIX)
fi
fi
if test "${use_tcl}" = "no" ; then
HAVE_TCL=""
else
HAVE_TCL=1
fi
AC_SUBST(HAVE_TCL)
##########
# Figure out what C libraries are required to compile programs
# that use "readline()" library.
@@ -445,11 +474,11 @@ TARGET_READLINE_INC=""
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
AC_ARG_ENABLE([editline],
[AS_HELP_STRING([--enable-editline],[enable BSD editline support])],
[AC_HELP_STRING([--enable-editline],[enable BSD editline support])],
[with_editline=$enableval],
[with_editline=auto])
AC_ARG_ENABLE([readline],
[AS_HELP_STRING([--disable-readline],[disable readline support])],
[AC_HELP_STRING([--disable-readline],[disable readline support])],
[with_readline=$enableval],
[with_readline=auto])
@@ -465,7 +494,7 @@ if test x"$with_readline" != xno; then
found="yes"
AC_ARG_WITH([readline-lib],
[AS_HELP_STRING([--with-readline-lib],[specify readline library])],
[AC_HELP_STRING([--with-readline-lib],[specify readline library])],
[with_readline_lib=$withval],
[with_readline_lib="auto"])
if test "x$with_readline_lib" = xauto; then
@@ -480,7 +509,7 @@ if test x"$with_readline" != xno; then
fi
AC_ARG_WITH([readline-inc],
[AS_HELP_STRING([--with-readline-inc],[specify readline include paths])],
[AC_HELP_STRING([--with-readline-inc],[specify readline include paths])],
[with_readline_inc=$withval],
[with_readline_inc="auto"])
if test "x$with_readline_inc" = xauto; then
@@ -511,28 +540,11 @@ if test x"$with_readline" != xno; then
TARGET_HAVE_READLINE=1
fi
fi
AC_ARG_WITH([linenoise],
[AS_HELP_STRING([--with-linenoise=DIR],[source directory for linenoise library])],
[with_linenoise=$withval],
[with_linenoise="no"])
if test "x$with_linenoise" != "xno"; then
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
TARGET_HAVE_LINENOISE=1
TARGET_READLINE_INC="-I${with_linenoise}"
TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
echo "using linenoise source code at ${with_linenoise}"
else
TARGET_HAVE_LINENOISE=0
echo "not using linenoise"
fi
AC_SUBST(TARGET_READLINE_LIBS)
AC_SUBST(TARGET_READLINE_INC)
AC_SUBST(TARGET_HAVE_READLINE)
AC_SUBST(TARGET_HAVE_EDITLINE)
AC_SUBST(TARGET_HAVE_LINENOISE)
##########
# Figure out what C libraries are required to compile programs
@@ -542,10 +554,10 @@ AC_SEARCH_LIBS(fdatasync, [rt])
#########
# check for debug enabled
AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
AC_MSG_CHECKING([build type])
if test "${enable_debug}" = "yes" ; then
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
AC_MSG_RESULT([debug])
else
TARGET_DEBUG="-DNDEBUG"
@@ -555,8 +567,7 @@ AC_SUBST(TARGET_DEBUG)
#########
# See whether we should use the amalgamation to build
AC_ARG_ENABLE(amalgamation, AS_HELP_STRING([--disable-amalgamation],
AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
[Disable the amalgamation and instead build all files separately]))
if test "${enable_amalgamation}" = "no" ; then
USE_AMALGAMATION=0
@@ -571,7 +582,7 @@ AC_SUBST(HAVE_ZLIB)
#########
# See whether we should allow loadable extensions
AC_ARG_ENABLE(load-extension, AS_HELP_STRING([--disable-load-extension],
AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
[Disable loading of external extensions]),,[enable_load_extension=yes])
if test "${enable_load_extension}" = "yes" ; then
OPT_FEATURE_FLAGS=""
@@ -584,7 +595,7 @@ fi
# Do we want to support math functions
#
AC_ARG_ENABLE(math,
AS_HELP_STRING([--disable-math],[Disable math functions]))
AC_HELP_STRING([--disable-math],[Disable math functions]))
AC_MSG_CHECKING([whether to support math functions])
if test "$enable_math" = "no"; then
AC_MSG_RESULT([no])
@@ -594,30 +605,18 @@ else
AC_SEARCH_LIBS(ceil, m)
fi
##########
# Do we want to support JSON functions
#
AC_ARG_ENABLE(json,
AS_HELP_STRING([--disable-json],[Disable JSON functions]))
AC_MSG_CHECKING([whether to support JSON functions])
if test "$enable_json" = "no"; then
AC_MSG_RESULT([no])
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON"
else
AC_MSG_RESULT([yes])
fi
########
# The --enable-all argument is short-hand to enable
# multiple extensions.
AC_ARG_ENABLE(all, AS_HELP_STRING([--enable-all],
[Enable FTS4, FTS5, Geopoly, RTree, Sessions]))
AC_ARG_ENABLE(all, AC_HELP_STRING([--enable-all],
[Enable FTS4, FTS5, Geopoly, JSON, RTree, Sessions]))
##########
# Do we want to support memsys3 and/or memsys5
#
AC_ARG_ENABLE(memsys5,
AS_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
AC_MSG_CHECKING([whether to support MEMSYS5])
if test "${enable_memsys5}" = "yes"; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
@@ -626,7 +625,7 @@ else
AC_MSG_RESULT([no])
fi
AC_ARG_ENABLE(memsys3,
AS_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
AC_MSG_CHECKING([whether to support MEMSYS3])
if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
@@ -637,7 +636,7 @@ fi
#########
# See whether we should enable Full Text Search extensions
AC_ARG_ENABLE(fts3, AS_HELP_STRING([--enable-fts3],
AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
[Enable the FTS3 extension]))
AC_MSG_CHECKING([whether to support FTS3])
if test "${enable_fts3}" = "yes" ; then
@@ -646,7 +645,7 @@ if test "${enable_fts3}" = "yes" ; then
else
AC_MSG_RESULT([no])
fi
AC_ARG_ENABLE(fts4, AS_HELP_STRING([--enable-fts4],
AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
[Enable the FTS4 extension]))
AC_MSG_CHECKING([whether to support FTS4])
if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -656,7 +655,7 @@ if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
else
AC_MSG_RESULT([no])
fi
AC_ARG_ENABLE(fts5, AS_HELP_STRING([--enable-fts5],
AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
[Enable the FTS5 extension]))
AC_MSG_CHECKING([whether to support FTS5])
if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -667,10 +666,21 @@ else
AC_MSG_RESULT([no])
fi
#########
# See whether we should enable JSON1
AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
AC_MSG_CHECKING([whether to support JSON])
if test "${enable_json1}" = "yes" -o "${enable_all}" = "yes" ; then
OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
fi
#########
# See whether we should enable the LIMIT clause on UPDATE and DELETE
# statements.
AC_ARG_ENABLE(update-limit, AS_HELP_STRING([--enable-update-limit],
AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
[Enable the UPDATE/DELETE LIMIT clause]))
AC_MSG_CHECKING([whether to support LIMIT on UPDATE and DELETE statements])
if test "${enable_update_limit}" = "yes" ; then
@@ -682,7 +692,7 @@ fi
#########
# See whether we should enable GEOPOLY
AC_ARG_ENABLE(geopoly, AS_HELP_STRING([--enable-geopoly],
AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
[Enable the GEOPOLY extension]),
[enable_geopoly=yes],[enable_geopoly=no])
AC_MSG_CHECKING([whether to support GEOPOLY])
@@ -696,7 +706,7 @@ fi
#########
# See whether we should enable RTREE
AC_ARG_ENABLE(rtree, AS_HELP_STRING([--enable-rtree],
AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
[Enable the RTREE extension]))
AC_MSG_CHECKING([whether to support RTREE])
if test "${enable_rtree}" = "yes" ; then
@@ -708,7 +718,7 @@ fi
#########
# See whether we should enable the SESSION extension
AC_ARG_ENABLE(session, AS_HELP_STRING([--enable-session],
AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
[Enable the SESSION extension]))
AC_MSG_CHECKING([whether to support SESSION])
if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -772,7 +782,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
#########
# See whether we should use GCOV
AC_ARG_ENABLE(gcov, AS_HELP_STRING([--enable-gcov],
AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
[Enable coverage testing using gcov]))
if test "${use_gcov}" = "yes" ; then
USE_GCOV=1
@@ -781,28 +791,16 @@ else
fi
AC_SUBST(USE_GCOV)
#########
# Enable/disabled amalagamation line macros
########
AMALGAMATION_LINE_MACROS=--linemacros=0
if test "${amalgamation_line_macros}" = "yes" ; then
AMALGAMATION_LINE_MACROS=--linemacros=1
fi
if test "${amalgamation_line_macros}" = "no" ; then
AMALGAMATION_LINE_MACROS=--linemacros=0
fi
AC_SUBST(AMALGAMATION_LINE_MACROS)
#########
# Output the config header
AC_CONFIG_HEADERS(sqlite_cfg.h)
AC_CONFIG_HEADERS(config.h)
#########
# Generate the output files.
#
AC_SUBST(BUILD_CFLAGS)
AC_CONFIG_FILES([
AC_OUTPUT([
Makefile
sqlite3.pc
])
AC_OUTPUT
-167
View File
@@ -1,167 +0,0 @@
# Notes On Compiling SQLite On Windows 11
Here are step-by-step instructions on how to build SQLite from
canonical source on a new Windows 11 PC, as of 2023-11-01:
1. Install Microsoft Visual Studio. The free "community edition"
will work fine. Do a standard install for C++ development.
SQLite only needs the
"cl" compiler and the "nmake" build tool.
2. Under the "Start" menu, find "All Apps" then go to "Visual Studio 20XX"
and find "x64 Native Tools Command Prompt for VS 20XX". Pin that
application to your task bar, as you will use it a lot. Bring up
an instance of this command prompt and do all of the subsequent steps
in that "x64 Native Tools" command prompt. (Or use "x86" if you want
a 32-bit build.) The subsequent steps will not work in a vanilla
DOS prompt. Nor will they work in PowerShell.
3. Install TCL development libraries. This note assumes that you will
install the TCL development libraries in the "`c:\Tcl`" directory.
Make adjustments
if you want TCL installed somewhere else. SQLite needs both the
"tclsh.exe" command-line tool as part of the build process, and
the "tcl86.lib" library in order to run tests. You will need
TCL version 8.6 or later.
<ol type="a">
<li>Get the TCL source archive, perhaps from
[https://www.tcl.tk/software/tcltk/download.html](https://www.tcl.tk/software/tcltk/download.html).
<li>Untar or unzip the source archive. CD into the "win/" subfolder
of the source tree.
<li>Run: `nmake /f makefile.vc release`
<li>Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl install`
<li>CD to `c:\Tcl\lib`. In that subfolder make a copy of the
"`tcl86t.lib`" file to the alternative name "`tcl86.lib`"
(omitting the second 't'). Leave the copy in the same directory
as the original.
<li>CD to `c:\Tcl\bin`. Make a copy of the "`tclsh86t.exe`"
file into "`tclsh.exe`" (without the "86t") in the same directory.
<li>Add `c:\Tcl\bin` to your %PATH%. To do this, go to Settings
and search for "path". Select "edit environment variables for
your account" and modify your default PATH accordingly.
You will need to close and reopen your command prompts after
making this change.
</ol>
4. Download the SQLite source tree and unpack it. CD into the
toplevel directory of the source tree.
5. Set the TCLDIR environment variable to point to your TCL installation.
Like this:
<ul>
<li> `set TCLDIR=c:\Tcl`
</ul>
6. Run the "`Makefile.msc`" makefile with an appropriate target.
Examples:
<ul>
<li> `nmake /f makefile.msc`
<li> `nmake /f makefile.msc sqlite3.c`
<li> `nmake /f makefile.msc sqlite3.exe`
<li> `nmake /f makefile.msc sqldiff.exe`
<li> `nmake /f makefile.msc tclextension-install`
<li> `nmake /f makefile.msc devtest`
<li> `nmake /f makefile.msc releasetest`
<li> `nmake /f makefile.msc sqlite3_analyzer.exe`
</ul>
It is not required that you run the "tclextension-install" target prior to
running tests. However, the tests will run more smoothly if you do.
The version of SQLite used for the TCL extension does *not* need to
correspond to the version of SQLite under test. So you can install the
SQLite TCL extension once, and then use it to test many different versions
of SQLite.
7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
commands work, add the DEBUG=3 argument to nmake. Like this:
<ul>
<li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
</ul>
## 32-bit Builds
Doing a 32-bit build is just like doing a 64-bit build with the
following minor changes:
1. Use the "x86 Native Tools Command Prompt" instead of
"x64 Native Tools Command Prompt". "**x86**" instead of "**x64**".
2. Use a different installation directory for TCL.
The recommended directory is `c:\tcl32`. Thus you end up
with two TCL builds:
<ul>
<li> `c:\tcl` &larr; 64-bit (the default)
<li> `c:\tcl32` &larr; 32-bit
</ul>
3. Ensure that `c:\tcl32\bin` comes before `c:\tcl\bin` on
your PATH environment variable. You can achieve this using
a command like:
<ul>
<li> `set PATH=c:\tcl32\bin;%PATH%`
</ul>
## Building a DLL
The command the developers use for building the deliverable DLL on the
[download page](https://sqlite.org/download.html) is as follows:
> ~~~~
nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
~~~~
That command generates both the sqlite3.dll and sqlite3.def files. The same
command works for both 32-bit and 64-bit builds.
## Statically Linking The TCL Library
Some utility programs associated with SQLite need to be linked
with TCL in order to function. The [sqlite3_analyzer.exe program](https://sqlite.org/sqlanalyze.html)
is an example. You can build as described above, and then
enter:
> ~~~~
nmake /f Makefile.msc sqlite3_analyzer.exe
~~~~
And you will end up with a working executable. However, that executable
will depend on having the "tcl86.dll" library somewhere on your %PATH%.
Use the following steps to build an executable that has the TCL library
statically linked so that it does not depend on separate DLL:
1. Use the appropriate "Command Prompt" window - either x86 or
x64, depending on whether you want a 32-bit or 64-bit executable.
2. Untar the TCL source tarball into a fresh directory. CD into
the "win/" subfolder.
3. Run: `nmake /f makefile.vc OPTS=nothreads,static shell`
4. CD into the "Release*" subfolder that is created (note the
wildcard - the full name of the directory might vary). There
you will find the "tcl86s.lib" file. Copy this file into the
same directory that you put the "tcl86.lib" on your initial
installation. (In this document, that directory is
"`C:\Tcl32\lib`" for 32-bit builds and
"`C:\Tcl\lib`" for 64-bit builds.)
5. CD into your SQLite source code directory and build the desired
utility program, but add the following extra arguments to the
nmake command line:
<blockquote><pre>
CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib"
</pre></blockquote>
<p>So, for example, to build a statically linked version of
sqlite3_analyzer.exe, you might type:
<blockquote><pre>
nmake /f Makefile.msc CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib" sqlite3_analyzer.exe
</pre></blockquote>
6. After your executable is built, you can verify that it does not
depend on the TCL DLL by running:
<blockquote><pre>
dumpbin /dependents sqlite3_analyzer.exe
</pre></blockquote>
-144
View File
@@ -1,144 +0,0 @@
# JSON Functions Enhancements (2022)
This document summaries enhancements to the SQLite JSON support added in
early 2022.
## 1.0 Change summary:
1. New **->** and **->>** operators that work like MySQL and PostgreSQL (PG).
2. JSON functions are built-in rather than being an extension. They
are included by default, but can be omitted using the
-DSQLITE_OMIT_JSON compile-time option.
## 2.0 New operators **->** and **->>**
The SQLite language adds two new binary operators **->** and **->>**.
Both operators are similar to json_extract(). The left operand is
JSON and the right operand is a JSON path expression (possibly abbreviated
for compatibility with PG - see below). So they are similar to a
two-argument call to json_extract().
The difference between -> and ->> (and json_extract()) is as follows:
* The -> operator always returns JSON.
* The ->> operator converts the answer into a primitive SQL datatype
such as TEXT, INTEGER, REAL, or NULL. If a JSON object or array
is selected, that object or array is rendered as text. If a JSON
value is selected, that value is converted into its corresponding
SQL type
* The json_extract() interface returns JSON when a JSON object or
array is selected, or a primitive SQL datatype when a JSON value
is selected. This is different from MySQL, in which json_extract()
always returns JSON, but the difference is retained because it has
worked that way for 6 years and changing it now would likely break
a lot of legacy code.
In MySQL and PG, the ->> operator always returns TEXT (or NULL) and never
INTEGER or REAL. This is due to limitations in the type handling capabilities
of those systems. In MySQL and PG, the result type a function or operator
may only depend on the type of its arguments, never the value of its arguments.
But the underlying JSON type depends on the value of the JSON path
expression, not the type of the JSON path expression (which is always TEXT).
Hence, the result type of ->> in MySQL and PG is unable to vary according
to the type of the JSON value being extracted.
The type system in SQLite is more general. Functions in SQLite are able
to return different datatypes depending on the value of their arguments.
So the ->> operator in SQLite is able to return TEXT, INTEGER, REAL, or NULL
depending on the JSON type of the value being extracted. This means that
the behavior of the ->> is slightly different in SQLite versus MySQL and PG
in that it will sometimes return INTEGER and REAL values, depending on its
inputs. It is possible to implement the ->> operator in SQLite so that it
always operates exactly like MySQL and PG and always returns TEXT or NULL,
but I have been unable to think of any situations where returning the
actual JSON value this would cause problems, so I'm including the enhanced
functionality in SQLite.
The table below attempts to summarize the differences between the
-> and ->> operators and the json_extract() function, for SQLite, MySQL,
and PG. JSON values are shown using their SQL text representation but
in a bold font.
<table border=1 cellpadding=5 cellspacing=0>
<tr><th>JSON<th>PATH<th>-&gt; operator<br>(all)<th>-&gt;&gt; operator<br>(MySQL/PG)
<th>-&gt;&gt; operator<br>(SQLite)<th>json_extract()<br>(SQLite)
<tr><td> **'{"a":123}'** <td>'$.a'<td> **'123'** <td> '123' <td> 123 <td> 123
<tr><td> **'{"a":4.5}'** <td>'$.a'<td> **'4.5'** <td> '4.5' <td> 4.5 <td> 4.5
<tr><td> **'{"a":"xyz"}'** <td>'$.a'<td> **'"xyz"'** <td> 'xyz' <td> 'xyz' <td> 'xyz'
<tr><td> **'{"a":null}'** <td>'$.a'<td> **'null'** <td> NULL <td> NULL <td> NULL
<tr><td> **'{"a":[6,7,8]}'** <td>'$.a'<td> **'[6,7,8]'** <td> '[6,7,8]' <td> '[6,7,8]' <td> **'[6,7,8]'**
<tr><td> **'{"a":{"x":9}}'** <td>'$.a'<td> **'{"x":9}'** <td> '{"x":9}' <td> '{"x":9}' <td> **'{"x":9}'**
<tr><td> **'{"b":999}'** <td>'$.a'<td> NULL <td> NULL <td> NULL <td> NULL
</table>
Important points about the table above:
* The -> operator always returns either JSON or NULL.
* The ->> operator never returns JSON. It always returns TEXT or NULL, or in the
case of SQLite, INTEGER or REAL.
* The MySQL json_extract() function works exactly the same
as the MySQL -> operator.
* The SQLite json_extract() operator works like -> for JSON objects and
arrays, and like ->> for JSON values.
* The -> operator works the same for all systems.
* The only difference in ->> between SQLite and other systems is that
when the JSON value is numeric, SQLite returns a numeric SQL value,
whereas the other systems return a text representation of the numeric
value.
### 2.1 Abbreviated JSON path expressions for PG compatibility
The table above always shows the full JSON path expression: '$.a'. But
PG does not accept this syntax. PG only allows a single JSON object label
name or a single integer array index. In order to provide compatibility
with PG, The -> and ->> operators in SQLite are extended to also support
a JSON object label or an integer array index for the right-hand side
operand, in addition to a full JSON path expression.
Thus, a -> or ->> operator that works on MySQL will work in
SQLite. And a -> or ->> operator that works in PG will work in SQLite.
But because SQLite supports the union of the disjoint capabilities of
MySQL and PG, there will always be -> and ->> operators that work in
SQLite that do not work in one of MySQL and PG. This is an unavoidable
consequence of the different syntax for -> and ->> in MySQL and PG.
In the following table, assume that "value1" is a JSON object and
"value2" is a JSON array.
<table border=1 cellpadding=5 cellspacing=0>
<tr><th>SQL expression <th>Works in MySQL?<th>Works in PG?<th>Works in SQLite
<tr><td>value1-&gt;'$.a' <td> yes <td> no <td> yes
<tr><td>value1-&gt;'a' <td> no <td> yes <td> yes
<tr><td>value2-&gt;'$[2]' <td> yes <td> no <td> yes
<tr><td>value2-&gt;2 <td> no <td> yes <td> yes
</table>
The abbreviated JSON path expressions only work for the -> and ->> operators
in SQLite. The json_extract() function, and all other built-in SQLite
JSON functions, continue to require complete JSON path expressions for their
PATH arguments.
## 3.0 JSON moved into the core
The JSON interface is now moved into the SQLite core.
When originally written in 2015, the JSON functions were an extension
that could be optionally included at compile-time, or loaded at run-time.
The implementation was in a source file named ext/misc/json1.c in the
source tree. JSON functions were only compiled in if the
-DSQLITE_ENABLE_JSON1 compile-time option was used.
After these enhancements, the JSON functions are now built-ins.
The source file that implements the JSON functions is moved to src/json.c.
No special compile-time options are needed to load JSON into the build.
Instead, there is a new -DSQLITE_OMIT_JSON compile-time option to leave
them out.
-290
View File
@@ -1,290 +0,0 @@
# The JSONB Format
This document describes SQLite's JSONB binary encoding of
JSON.
## 1.0 What Is JSONB?
Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an
alternative binary encoding of JSON which we call "JSONB". JSONB is
a binary format that stored as a BLOB.
The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB
is both slightly smaller (by between 5% and 10% in most cases) and
can be processed in less than half the number of CPU cycles. The built-in
[JSON SQL functions] of SQLite can accept either ordinary text JSON
or the binary JSONB encoding for any of their JSON inputs.
The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the
on-disk format for SQLite's JSONB is not the same as PostgreSQL's.
The two formats have the same name, but they have wildly different internal
representations and are not in any way binary compatible.
The central idea behind this JSONB specification is that each element
begins with a header that includes the size and type of that element.
The header takes the place of punctuation such as double-quotes,
curly-brackes, square-brackets, commas, and colons. Since the size
and type of each element is contained in its header, the element can
be read faster since it is no longer necessary to carefully scan forward
looking for the closing delimiter. The payload of JSONB is the same
as for corresponding text JSON. The same payload bytes occur in the
same order. The only real difference between JSONB and ordinary text
JSON is that JSONB includes a binary header on
each element and omits delimiter and separator punctuation.
### 1.1 Internal Use Only
The details of the JSONB are not intended to be visible to application
developers. Application developers should look at JSONB as an opaque BLOB
used internally by SQLite. Nevertheless, we want the format to be backwards
compatible across all future versions of SQLite. To that end, the format
is documented by this file in the source tree. But this file should be
used only by SQLite core developers, not by developers of applications
that only use SQLite.
## 2.0 The Purpose Of This Document
JSONB is not intended as an external format to be used by
applications. JSONB is designed for internal use by SQLite only.
Programmers do not need to understand the JSONB format in order to
use it effectively.
Applications should access JSONB only through the [JSON SQL functions],
not by looking at individual bytes of the BLOB.
However, JSONB is intended to be portable and backwards compatible
for all future versions of SQLite. In other words, you should not have
to export and reimport your SQLite database files when you upgrade to
a newer SQLite version. For that reason, the JSONB format needs to
be well-defined.
This document is therefore similar in purpose to the
[SQLite database file format] document that describes the on-disk
format of an SQLite database file. Applications are not expected
to directly read and write the bits and bytes of SQLite database files.
The SQLite database file format is carefully documented so that it
can be stable and enduring. In the same way, the JSONB representation
of JSON is documented here so that it too can be stable and enduring,
not so that applications can read or writes individual bytes.
## 3.0 Encoding
JSONB is a direct translation of the underlying text JSON. The difference
is that JSONB uses a binary encoding that is faster to parse compared to
the detailed syntax of text JSON.
Each JSON element is encoded as a header and a payload. The header
determines type of element (string, numeric, boolean, null, object, or
array) and the size of the payload. The header can be between 1 and
9 bytes in size. The payload can be any size from zero bytes up to the
maximum allowed BLOB size.
### 3.1 Payload Size
The upper four bits of the first byte of the header determine size of the
header and possibly also the size of the payload.
If the upper four bits have a value between 0 and 11, then the header is
exactly one byte in size and the payload size is determined by those
upper four bits. If the upper four bits have a value between 12 and 15,
that means that the total header size is 2, 3, 5, or 9 bytes and the
payload size is unsigned big-endian integer that is contained in the
subsequent bytes. The size integer is the one byte that following the
initial header byte if the upper four bits
are 12, two bytes if the upper bits are 13, four bytes if the upper bits
are 14, and eight bytes if the upper bits are 15. The current design
of SQLite does not support BLOB values larger than 2GiB, so the eight-byte
variant of the payload size integer will never be used by the current code.
The eight-byte payload size integer is included in the specification
to allow for future expansion.
The header for an element does *not* need to be in its simplest
form. For example, consider the JSON numeric value "`1`".
That element can be encode in five different ways:
* `0x13 0x31`
* `0xc3 0x01 0x31`
* `0xd3 0x00 0x01 0x31`
* `0xe3 0x00 0x00 0x00 0x01 0x31`
* `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31`
The shortest encoding is preferred, of course, and usually happens with
primitive elements such as numbers. However the total size of an array
or object might not be known exactly when the header of the element is
first generated. It is convenient to reserve space for the largest
possible header and then go back and fill in the correct payload size
at the end. This technique can result in array or object headers that
are larger than absolutely necessary.
### 3.2 Element Type
The least-significant four bits of the first byte of the header (the first
byte masked against 0x0f) determine element type. The following codes are
used:
<ol>
<li type="0"><p><b>NULL</b> &rarr;
The element is a JSON "null". The payload size for a true JSON NULL must
must be zero. Future versions of SQLite might extend the JSONB format
with elements that have a zero element type but a non-zero size. In that
way, legacy versions of SQLite will interpret the element as a NULL
for backwards compatibility while newer versions will interpret the
element in some other way.
<li value="1"><p><b>TRUE</b> &rarr;
The element is a JSON "true". The payload size must be zero for a actual
"true" value. Elements with type 1 and a non-zero payload size are
reserved for future expansion. Legacy implementations that see an element
type of 1 with a non-zero payload size should continue to interpret that
element as "true" for compatibility.
<li value="2"><p><b>FALSE</b> &rarr;
The element is a JSON "false". The payload size must be zero for a actual
"false" value. Elements with type 2 and a non-zero payload size are
reserved for future expansion. Legacy implementations that see an element
type of 2 with a non-zero payload size should continue to interpret that
element as "false" for compatibility.
<li value="3"><p><b>INT</b> &rarr;
The element is a JSON integer value in the canonical
RFC 8259 format, without extensions. The payload is the ASCII
text representation of that numeric value.
<li value="4"><p><b>INT5</b> &rarr;
The element is a JSON integer value that is not in the
canonical format. The payload is the ASCII
text representation of that numeric value. Because the payload is in a
non-standard format, it will need to be translated when the JSONB is
converted into RFC 8259 text JSON.
<li value="5"><p><b>FLOAT</b> &rarr;
The element is a JSON floating-point value in the canonical
RFC 8259 format, without extensions. The payload is the ASCII
text representation of that numeric value.
<li value="6"><p><b>FLOAT5</b> &rarr;
The element is a JSON floating-point value that is not in the
canonical format. The payload is the ASCII
text representation of that numeric value. Because the payload is in a
non-standard format, it will need to be translated when the JSONB is
converted into RFC 8259 text JSON.
<li value="7"><p><b>TEXT</b> &rarr;
The element is a JSON string value that does not contain
any escapes nor any characters that need to be escaped for either SQL or
JSON. The payload is the UTF8 text representation of the string value.
The payload does <i>not</i> include string delimiters.
<li value="8"><p><b>TEXTJ</b> &rarr;
The element is a JSON string value that contains
RFC 8259 character escapes (such as "<tt>\n</tt>" or "<tt>\u0020</tt>").
Those escapes will need to be translated into actual UTF8 if this element
is [json_extract|extracted] into SQL.
The payload is the UTF8 text representation of the escaped string value.
The payload does <i>not</i> include string delimiters.
<li value="9"><p><b>TEXT5</b> &rarr;
The element is a JSON string value that contains
character escapes, including some character escapes that part of JSON5
and which are not found in the canonical RFC 8259 spec.
Those escapes will need to be translated into standard JSON prior to
rendering the JSON as text, or into their actual UTF8 characters if this
element is [json_extract|extracted] into SQL.
The payload is the UTF8 text representation of the escaped string value.
The payload does <i>not</i> include string delimiters.
<li value="10"><p><b>TEXTRAW</b> &rarr;
The element is a JSON string value that contains
UTF8 characters that need to be escaped if this string is rendered into
standard JSON text.
The payload does <i>not</i> include string delimiters.
<li value="11"><p><b>ARRAY</b> &rarr;
The element is a JSON array. The payload contains
JSONB elements that comprise values contained within the array.
<li value="12"><p><b>OBJECT</b> &rarr;
The element is a JSON object. The payload contains
pairs of JSONB elements that comprise entries for the JSON object.
The first element in each pair must be a string (types 7 through 10).
The second element of each pair may be any types, including nested
arrays or objects.
<li value="13"><p><b>RESERVED-13</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
<li value="14"><p><b>RESERVED-14</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
<li value="15"><p><b>RESERVED-15</b> &rarr;
Reserved for future expansion. Legacy implements that encounter this
element type should raise an error.
</ol>
Element types outside the range of 0 to 12 are reserved for future
expansion. The current implement raises an error if see an element type
other than those listed above. However, future versions of SQLite might
use of the three remaining element types to implement indexing or similar
optimizations, to speed up lookup against large JSON arrays and/or objects.
### 3.3 Design Rationale For Element Types
A key goal of JSONB is that it should be quick to translate
to and from text JSON and/or be constructed from SQL values.
When converting from text into JSONB, we do not want the
converter subroutine to burn CPU cycles converting elements
values into some standard format which might never be used.
Format conversion is "lazy" - it is deferred until actually
needed. This has implications for the JSONB format design:
1. Numeric values are stored as text, not a numbers. The values are
a direct copy of the text JSON values from which they are derived.
2. There are multiple element types depending on the details of value
formats. For example, INT is used for pure RFC-8259 integer
literals and INT5 exists for JSON5 extensions such as hexadecimal
notation. FLOAT is used for pure RFC-8259 floating point literals
and FLOAT5 is used for JSON5 extensions. There are four different
representations of strings, depending on where the string came from
and how special characters within the string are escaped.
A second goal of JSONB is that it should be capable of serving as the
"parse tree" for JSON when a JSON value is being processed by the
various [JSON SQL functions] built into SQLite. Before JSONB was
developed, operations such [json_replace()] and [json_patch()]
and similar worked in three stages:
1. Translate the text JSON into a internal format that is
easier to scan and edit.
2. Perform the requested operation on the JSON.
3. Translate the internal format back into text.
JSONB seeks to serve as the internal format directly - bypassing
the first and third stages of that process. Since most of the CPU
cycles are spent on the first and third stages, that suggests that
JSONB processing will be much faster than text JSON processing.
So when processing JSONB, only the second stage of the three-stage
process is required. But when processing text JSON, it is still necessary
to do stages one and three. If JSONB is to be used as the internal
binary representation, this is yet another reason to store numeric
values as text. Storing numbers as text minimizes the amount of
conversion work needed for stages one and three. This is also why
there are four different representations of text in JSONB. Different
text representations are used for text coming from different sources
(RFC-8259 JSON, JSON5, or SQL string values) and conversions only
happen if and when they are actually needed.
### 3.4 Valid JSONB BLOBs
A valid JSONB BLOB consists of a single JSON element. The element must
exactly fill the BLOB. This one element is often a JSON object or array
and those usually contain additional elements as its payload, but the
element can be a primite value such a string, number, boolean, or null.
When the built-in JSON functions are attempting to determine if a BLOB
argument is a JSONB or just a random BLOB, they look at the header of
the outer element to see that it is well-formed and that the element
completely fills the BLOB. If these conditions are met, then the BLOB
is accepted as a JSONB value.
+3 -21
View File
@@ -322,7 +322,7 @@ These are the steps:
<ul>
<li> Declare a local variable of type "yyParser"
<li> Initialize the variable using ParseInit()
<li> Pass a pointer to the variable in calls to Parse()
<li> Pass a pointer to the variable in calls ot Parse()
<li> Deallocate substructure in the parse variable using ParseFinalize().
</ul>
@@ -683,7 +683,6 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt><a href='#pifdef'>%endif</a></tt>
<li><tt><a href='#extraarg'>%extra_argument</a></tt>
<li><tt><a href='#pfallback'>%fallback</a></tt>
<li><tt><a href='#reallc'>%free</a></tt>
<li><tt><a href='#pifdef'>%if</a></tt>
<li><tt><a href='#pifdef'>%ifdef</a></tt>
<li><tt><a href='#pifdef'>%ifndef</a></tt>
@@ -694,7 +693,6 @@ other than that, the order of directives in Lemon is arbitrary.</p>
<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
<li><tt><a href='#pright'>%right</a></tt>
<li><tt><a href='#reallc'>%realloc</a></tt>
<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
<li><tt><a href='#stack_size'>%stack_size</a></tt>
<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
@@ -1202,21 +1200,6 @@ match any input token.</p>
the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no alternatives.</p>
<a id='reallc'></a>
<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4>
<p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function
that allocate and free heap memory. The signatures of these functions
should be the same as the realloc() and free() functions from the standard
C library.
<p>If both of these functions are defined
then these functions are used to allocate and free
memory for supplemental parser stack space, if the initial
parse stack space is exceeded. The initial parser stack size
is specified by either <tt>%stack_size</tt> or the
-DYYSTACKDEPTH compile-time flag.
<a id='errors'></a>
<h2>5.0 Error Processing</h2>
@@ -1241,14 +1224,13 @@ to begin parsing a new file. This is what will happen at the very
first syntax error, of course, if there are no instances of the
"error" non-terminal in your grammar.</p>
<a id='history'></a>
<h2>6.0 History of Lemon</h2>
<p>Lemon was originally written by Richard Hipp sometime in the late
1980s on a Sun4 Workstation using K&amp;R C.
There was a companion LL(1) parser generator program named "Lime".
The Lime source code has been lost.</p>
There was a companion LL(1) parser generator program named "Lime", the
source code to which as been lost.</p>
<p>The lemon.c source file was originally many separate files that were
compiled together to generate the "lemon" executable. Sometime in the
-356
View File
@@ -1,356 +0,0 @@
# The testrunner.tcl Script
<ul type=none>
<li> 1. <a href=#overview>Overview</a>
<li> 2. <a href=#binary_tests>Binary Tests</a>
<ul type=none>
<li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
<li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
<li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
</ul>
<li> 3. <a href=#source_code_tests>Source Tests</a>
<ul type=none>
<li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
<li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
<li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
</ul>
<li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
<li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
</ul>
<a name=overview></a>
# 1. Overview
testrunner.tcl is a Tcl script used to run multiple SQLite tests using
multiple jobs. It supports the following types of tests:
* Tcl test scripts.
* Tests run with `make` commands. Examples:
- `make mdevtest`
- `make releasetest`
- `make sdevtest`
- `make testrunner`
testrunner.tcl pipes the output of all tests and builds run into log file
**testrunner.log**, created in the current working directory. Search this
file to find details of errors. Suggested search commands:
* `grep "^!" testrunner.log`
* `grep failed testrunner.log`
testrunner.tcl also populates SQLite database **testrunner.db**. This database
contains details of all tests run, running and to be run. A useful query
might be:
```
SELECT * FROM script WHERE state='failed'
```
Running the command:
```
./testfixture $(TESTDIR)/testrunner.tcl status
```
in the directory containing the testrunner.db database runs various queries
to produce a succinct report on the state of a running testrunner.tcl script.
Running:
```
watch ./testfixture $(TESTDIR)/testrunner.tcl status
```
in another terminal is a good way to keep an eye on a long running test.
Sometimes testrunner.tcl uses the `testfixture` binary that it is run with
to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
other binaries in specific configurations to test (see "Source Tests").
<a name=binary_tests></a>
# 2. Binary Tests
The commands described in this section all run various combinations of the Tcl
test scripts using the `testfixture` binary used to run the testrunner.tcl
script (i.e. they do not invoke the compiler to build new binaries, or the
`make` command to run tests that are not Tcl scripts). The procedure to run
these tests is therefore:
1. Build the "testfixture" (or "testfixture.exe" for windows) binary using
whatever method seems convenient.
2. Test the binary built in step 1 by running testrunner.tcl with it,
perhaps with various options.
The following sub-sections describe the various options that can be
passed to testrunner.tcl to test binary testfixture builds.
<a name=organization_tests></a>
## 2.1. Organization of Tcl Tests
Tcl tests are stored in files that match the pattern *\*.test*. They are
found in both the $TOP/test/ directory, and in the various sub-directories
of the $TOP/ext/ directory of the source tree. Not all *\*.test* files
contain Tcl tests - a handful are Tcl scripts designed to invoke other
*\*.test* files.
The **veryquick** set of tests is a subset of all Tcl test scripts in the
source tree. In includes most tests, but excludes some that are very slow.
Almost all fault-injection tests (those that test the response of the library
to OOM or IO errors) are excluded. It is defined in source file
*test/permutations.test*.
The **full** set of tests includes all Tcl test scripts in the source tree.
To run a "full" test is to run all Tcl test scripts that can be found in the
source tree.
File *permutations.test* defines various test "permutations". A permutation
consists of:
* A subset of Tcl test scripts, and
* Runtime configuration to apply before running each test script
(e.g. enabling auto-vacuum, or disable lookaside).
Running **all** tests is to run all tests in the full test set, plus a dozen
or so permutations. The specific permutations that are run as part of "all"
are defined in file *testrunner_data.tcl*.
<a name=run_tests></a>
## 2.2. Commands to Run Tests
To run the "veryquick" test set, use either of the following:
```
./testfixture $TESTDIR/testrunner.tcl
./testfixture $TESTDIR/testrunner.tcl veryquick
```
To run the "full" test suite:
```
./testfixture $TESTDIR/testrunner.tcl full
```
To run the subset of the "full" test suite for which the test file name matches
a specified pattern (e.g. all tests that start with "fts5"), either of:
```
./testfixture $TESTDIR/testrunner.tcl fts5%
./testfixture $TESTDIR/testrunner.tcl 'fts5*'
```
Strictly speaking, for a test to be run the pattern must match the script
filename, not including the directory, using the rules of Tcl's
\[string match\] command. Except that before the matching is done, any "%"
characters specified as part of the pattern are transformed to "\*".
To run "all" tests (full + permutations):
```
./testfixture $TESTDIR/testrunner.tcl all
```
<a name=binary_test_failures></a>
## 2.3. Investigating Binary Test Failures
If a test fails, testrunner.tcl reports name of the Tcl test script and, if
applicable, the name of the permutation, to stdout. This information can also
be retrieved from either *testrunner.log* or *testrunner.db*.
If there is no permutation, the individual test script may be run with:
```
./testfixture $PATH_TO_SCRIPT
```
Or, if the failure occured as part of a permutation:
```
./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT
```
TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
<a name=source_code_tests></a>
# 3. Source Code Tests
The commands described in this section invoke the C compiler to build
binaries from the source tree, then use those binaries to run Tcl and
other tests. The advantages of this are that:
* it is possible to test multiple build configurations with a single
command, and
* it ensures that tests are always run using binaries created with the
same set of compiler options.
The testrunner.tcl commands described in this section may be run using
either a *testfixture* (or testfixture.exe) build, or with any other Tcl
shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
TODO: ./configure + Makefile.msc build systems.
<a name=commands_to_run_tests></a>
## 3.1. Commands to Run SQLite Tests
The **mdevtest** command is equivalent to running the veryquick tests and
the `make fuzztest` target once for each of two --enable-all builds - one
with debugging enabled and one without:
```
tclsh $TESTDIR/testrunner.tcl mdevtest
```
In other words, it is equivalent to running:
```
$TOP/configure --enable-all --enable-debug
make fuzztest
make testfixture
./testfixture $TOP/test/testrunner.tcl veryquick
# Then, after removing files created by the tests above:
$TOP/configure --enable-all OPTS="-O0"
make fuzztest
make testfixture
./testfixture $TOP/test/testrunner.tcl veryquick
```
The **sdevtest** command is identical to the mdevtest command, except that the
second of the two builds is a sanitizer build. Specifically, this means that
OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0":
```
tclsh $TESTDIR/testrunner.tcl sdevtest
```
The **release** command runs lots of tests under lots of builds. It runs
different combinations of builds and tests depending on whether it is run
on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details
of the specific tests run.
```
tclsh $TESTDIR/testrunner.tcl release
```
As with <a href=#source code tests>source code tests</a>, one or more patterns
may be appended to any of the above commands (mdevtest, sdevtest or release).
In that case only Tcl tests (no fuzz or other tests) that match the specified
pattern are run. For example, to run the just the Tcl rtree tests in all
builds and configurations supported by "release":
```
tclsh $TESTDIR/testrunner.tcl release rtree%
```
<a name=zipvfs_tests></a>
## 3.2. Running ZipVFS Tests
testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
tests from the Zipvfs project with the following command:
```
tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS
```
This can be combined with any of "mdevtest", "sdevtest" or "release" to
test both SQLite and Zipvfs with a single command:
```
tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
```
<a name=source_code_test_failures></a>
## 3.3. Investigating Source Code Test Failures
Investigating a test failure that occurs during source code testing is a
two step process:
1. Recreating the build configuration in which the test failed, and
2. Re-running the actual test.
To recreate a build configuration, use the testrunner.tcl **script** command
to create a build script. A build script is a bash script on Linux or OSX, or
a dos \*.bat file on windows. For example:
```
# Create a script that recreates build configuration "Device-One" on
# Linux or OSX:
tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh
# Create a script that recreates build configuration "Have-Not" on Windows:
tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat
```
The generated bash or \*.bat file script accepts a single argument - a makefile
target to build. This may be used either to run a `make` command test directly,
or else to build a testfixture (or testfixture.exe) binary with which to
run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
<a name=testrunner_options></a>
# 4. Extra testrunner.tcl Options
The testrunner.tcl script options in this section may be used with both source
code and binary tests.
The **--buildonly** option instructs testrunner.tcl just to build the binaries
required by a test, not to run any actual tests. For example:
```
# Build binaries required by release test.
tclsh $TESTDIR/testrunner.tcl --buildonly release"
```
The **--dryrun** option prevents testrunner.tcl from building any binaries
or running any tests. Instead, it just writes the shell commands that it
would normally execute into the testrunner.log file. Example:
```
# Log the shell commmands that make up the mdevtest test.
tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
```
The **--explain** option is similar to --dryrun in that it prevents testrunner.tcl
from building any binaries or running any tests. The difference is that --explain
prints on standard output a human-readable summary of all the builds and tests that
would have been run.
```
# Show what builds and tests would have been run
tclsh $TESTDIR/testrunner.tcl --explain mdevtest
```
<a name=cpu_cores></a>
# 5. Controlling CPU Core Utilization
When running either binary or source code tests, testrunner.tcl reports the
number of jobs it intends to use to stdout. e.g.
```
$ ./testfixture $TESTDIR/testrunner.tcl
splitting work across 16 jobs
... more output ...
```
By default, testfixture.tcl attempts to set the number of jobs to the number
of real cores on the machine. This can be overridden using the "--jobs" (or -j)
switch:
```
$ ./testfixture $TESTDIR/testrunner.tcl --jobs 8
splitting work across 8 jobs
... more output ...
```
The number of jobs may also be changed while an instance of testrunner.tcl is
running by exucuting the following command from the directory containing the
testrunner.log and testrunner.db files:
```
$ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS
```
-773
View File
@@ -1,773 +0,0 @@
/*
** 2023 November 4
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
********************************************************************************
** This file implements various interfaces used for console and stream I/O
** by the SQLite project command-line tools, as explained in console_io.h .
** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there.
*/
#ifndef SQLITE_CDECL
# define SQLITE_CDECL
#endif
#ifndef SHELL_NO_SYSINC
# include <stdarg.h>
# include <string.h>
# include <stdlib.h>
# include <limits.h>
# include <assert.h>
# include "sqlite3.h"
#endif
#ifndef HAVE_CONSOLE_IO_H
# include "console_io.h"
#endif
#if defined(_MSC_VER)
# pragma warning(disable : 4204)
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
# ifndef SHELL_NO_SYSINC
# include <io.h>
# include <fcntl.h>
# undef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# include <windows.h>
# endif
# define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */
# else
# ifndef SHELL_NO_SYSINC
# include <unistd.h>
# endif
# define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */
# endif
#else
# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
#endif
#if CIO_WIN_WC_XLATE
static HANDLE handleOfFile(FILE *pf){
int fileDesc = _fileno(pf);
union { intptr_t osfh; HANDLE fh; } fid = {
(fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE
};
return fid.fh;
}
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
typedef struct PerStreamTags {
# if CIO_WIN_WC_XLATE
HANDLE hx;
DWORD consMode;
char acIncomplete[4];
# else
short reachesConsole;
# endif
FILE *pf;
} PerStreamTags;
/* Define NULL-like value for things which can validly be 0. */
# define SHELL_INVALID_FILE_PTR ((FILE *)~0)
# if CIO_WIN_WC_XLATE
# define SHELL_INVALID_CONS_MODE 0xFFFF0000
# endif
# if CIO_WIN_WC_XLATE
# define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \
{0,0,0,0}, SHELL_INVALID_FILE_PTR }
# else
# define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR }
# endif
/* Quickly say whether a known output is going to the console. */
# if CIO_WIN_WC_XLATE
static short pstReachesConsole(PerStreamTags *ppst){
return (ppst->hx != INVALID_HANDLE_VALUE);
}
# else
# define pstReachesConsole(ppst) 0
# endif
# if CIO_WIN_WC_XLATE
static void restoreConsoleArb(PerStreamTags *ppst){
if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode);
}
# else
# define restoreConsoleArb(ppst)
# endif
/* Say whether FILE* appears to be a console, collect associated info. */
static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
# if CIO_WIN_WC_XLATE
short rv = 0;
DWORD dwCM = SHELL_INVALID_CONS_MODE;
HANDLE fh = handleOfFile(pf);
ppst->pf = pf;
if( INVALID_HANDLE_VALUE != fh ){
rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM));
}
ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE;
ppst->consMode = dwCM;
return rv;
# else
ppst->pf = pf;
ppst->reachesConsole = ( (short)isatty(fileno(pf)) );
return ppst->reachesConsole;
# endif
}
# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4)
# endif
# if CIO_WIN_WC_XLATE
/* Define console modes for use with the Windows Console API. */
# define SHELL_CONI_MODE \
(ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
| ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
# define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
| ENABLE_VIRTUAL_TERMINAL_PROCESSING)
# endif
typedef struct ConsoleInfo {
PerStreamTags pstSetup[3];
PerStreamTags pstDesignated[3];
StreamsAreConsole sacSetup;
} ConsoleInfo;
static short isValidStreamInfo(PerStreamTags *ppst){
return (ppst->pf != SHELL_INVALID_FILE_PTR);
}
static ConsoleInfo consoleInfo = {
{ /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
{ /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
SAC_NoConsole /* sacSetup */
};
SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0;
# if CIO_WIN_WC_XLATE
static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){
if( pstReachesConsole(ppst) ){
DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE;
SetConsoleMode(ppst->hx, cm);
}
}
# else
# define maybeSetupAsConsole(ppst,odir)
# endif
SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){
# if CIO_WIN_WC_XLATE
int ix = 0;
while( ix < 6 ){
PerStreamTags *ppst = (ix<3)?
&consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3];
maybeSetupAsConsole(ppst, (ix % 3)>0);
++ix;
}
# endif
}
SQLITE_INTERNAL_LINKAGE StreamsAreConsole
consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){
StreamsAreConsole rv = SAC_NoConsole;
FILE* apf[3] = { pfIn, pfOut, pfErr };
int ix;
for( ix = 2; ix >= 0; --ix ){
PerStreamTags *ppst = &consoleInfo.pstSetup[ix];
if( streamOfConsole(apf[ix], ppst) ){
rv |= (SAC_InConsole<<ix);
}
consoleInfo.pstDesignated[ix] = *ppst;
if( ix > 0 ) fflush(apf[ix]);
}
consoleInfo.sacSetup = rv;
consoleRenewSetup();
return rv;
}
SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){
# if CIO_WIN_WC_XLATE
static ConsoleInfo *pci = &consoleInfo;
if( pci->sacSetup ){
int ix;
for( ix=0; ix<3; ++ix ){
if( pci->sacSetup & (SAC_InConsole<<ix) ){
PerStreamTags *ppst = &pci->pstSetup[ix];
SetConsoleMode(ppst->hx, ppst->consMode);
}
}
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifdef SQLITE_CIO_INPUT_REDIR
/* Say whether given FILE* is among those known, via either
** consoleClassifySetup() or set{Output,Error}Stream, as
** readable, and return an associated PerStreamTags pointer
** if so. Otherwise, return 0.
*/
static PerStreamTags * isKnownReadable(FILE *pf){
static PerStreamTags *apst[] = {
&consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0
};
int ix = 0;
do {
if( apst[ix]->pf == pf ) break;
} while( apst[++ix] != 0 );
return apst[ix];
}
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
/* Say whether given FILE* is among those known, via either
** consoleClassifySetup() or set{Output,Error}Stream, as
** writable, and return an associated PerStreamTags pointer
** if so. Otherwise, return 0.
*/
static PerStreamTags * isKnownWritable(FILE *pf){
static PerStreamTags *apst[] = {
&consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2],
&consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0
};
int ix = 0;
do {
if( apst[ix]->pf == pf ) break;
} while( apst[++ix] != 0 );
return apst[ix];
}
static FILE *designateEmitStream(FILE *pf, unsigned chix){
FILE *rv = consoleInfo.pstDesignated[chix].pf;
if( pf == invalidFileStream ) return rv;
else{
/* Setting a possibly new output stream. */
PerStreamTags *ppst = isKnownWritable(pf);
if( ppst != 0 ){
PerStreamTags pst = *ppst;
consoleInfo.pstDesignated[chix] = pst;
}else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]);
}
return rv;
}
SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){
return designateEmitStream(pf, 1);
}
# ifdef CONSIO_SET_ERROR_STREAM
SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){
return designateEmitStream(pf, 2);
}
# endif
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifndef SQLITE_CIO_NO_SETMODE
# if CIO_WIN_WC_XLATE
static void setModeFlushQ(FILE *pf, short bFlush, int mode){
if( bFlush ) fflush(pf);
_setmode(_fileno(pf), mode);
}
# else
# define setModeFlushQ(f, b, m) if(b) fflush(f)
# endif
SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){
setModeFlushQ(pf, bFlush, _O_BINARY);
}
SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){
setModeFlushQ(pf, bFlush, _O_TEXT);
}
# undef setModeFlushQ
#else /* defined(SQLITE_CIO_NO_SETMODE) */
# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
#endif /* defined(SQLITE_CIO_NO_SETMODE) */
#ifndef SQLITE_CIO_NO_TRANSLATE
# if CIO_WIN_WC_XLATE
/* Write buffer cBuf as output to stream known to reach console,
** limited to ncTake char's. Return ncTake on success, else 0. */
static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){
int rv = 0;
if( z!=NULL ){
int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0);
if( nwc > 0 ){
WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR));
if( zw!=NULL ){
nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc);
if( nwc > 0 ){
/* Translation from UTF-8 to UTF-16, then WCHARs out. */
if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){
rv = ncTake;
}
}
sqlite3_free(zw);
}
}
}
return rv;
}
/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */
static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){
char *z = sqlite3_vmprintf(zFormat, ap);
if( z ){
int rv = conZstrEmit(ppst, z, (int)strlen(z));
sqlite3_free(z);
return rv;
}else return 0;
}
# endif /* CIO_WIN_WC_XLATE */
# ifdef CONSIO_GET_EMIT_STREAM
static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix,
PerStreamTags *ppst){
PerStreamTags *rv = isKnownWritable(pf);
short isValid = (rv!=0)? isValidStreamInfo(rv) : 0;
if( rv != 0 && isValid ) return rv;
streamOfConsole(pf, ppst);
return ppst;
}
# endif
/* Get stream info, either for designated output or error stream when
** chix equals 1 or 2, or for an arbitrary stream when chix == 0.
** In either case, ppst references a caller-owned PerStreamTags
** struct which may be filled in if none of the known writable
** streams is being held by consoleInfo. The ppf parameter is a
** byref output when chix!=0 and a byref input when chix==0.
*/
static PerStreamTags *
getEmitStreamInfo(unsigned chix, PerStreamTags *ppst,
/* in/out */ FILE **ppf){
PerStreamTags *ppstTry;
FILE *pfEmit;
if( chix > 0 ){
ppstTry = &consoleInfo.pstDesignated[chix];
if( !isValidStreamInfo(ppstTry) ){
ppstTry = &consoleInfo.pstSetup[chix];
pfEmit = ppst->pf;
}else pfEmit = ppstTry->pf;
if( !isValidStreamInfo(ppstTry) ){
pfEmit = (chix > 1)? stderr : stdout;
ppstTry = ppst;
streamOfConsole(pfEmit, ppstTry);
}
*ppf = pfEmit;
}else{
ppstTry = isKnownWritable(*ppf);
if( ppstTry != 0 ) return ppstTry;
streamOfConsole(*ppf, ppst);
return ppst;
}
return ppstTry;
}
SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){
va_list ap;
int rv;
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
rv = conioVmPrintf(ppst, zFormat, ap);
}else{
# endif
rv = vfprintf(pfOut, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){
va_list ap;
int rv;
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# else
getEmitStreamInfo(2, &pst, &pfErr);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
rv = conioVmPrintf(ppst, zFormat, ap);
}else{
# endif
rv = vfprintf(pfErr, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){
va_list ap;
int rv;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
# else
getEmitStreamInfo(0, &pst, &pfO);
# endif
assert(zFormat!=0);
va_start(ap, zFormat);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
maybeSetupAsConsole(ppst, 1);
rv = conioVmPrintf(ppst, zFormat, ap);
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
}else{
# endif
rv = vfprintf(pfO, zFormat, ap);
# if CIO_WIN_WC_XLATE
}
# endif
va_end(ap);
return rv;
}
SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
# else
getEmitStreamInfo(0, &pst, &pfO);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
int rv;
maybeSetupAsConsole(ppst, 1);
rv = conZstrEmit(ppst, z, (int)strlen(z));
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
return rv;
}else {
# endif
return (fputs(z, pfO)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# else
getEmitStreamInfo(2, &pst, &pfErr);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
else {
# endif
return (fputs(z, pfErr)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
assert(z!=0);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
else {
# endif
return (fputs(z, pfOut)<0)? 0 : (int)strlen(z);
# if CIO_WIN_WC_XLATE
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE))
/* Skip over as much z[] input char sequence as is valid UTF-8,
** limited per nAccept char's or whole characters and containing
** no char cn such that ((1<<cn) & ccm)!=0. On return, the
** sequence z:return (inclusive:exclusive) is validated UTF-8.
** Limit: nAccept>=0 => char count, nAccept<0 => character
*/
SQLITE_INTERNAL_LINKAGE const char*
zSkipValidUtf8(const char *z, int nAccept, long ccm){
int ng = (nAccept<0)? -nAccept : 0;
const char *pcLimit = (nAccept>=0)? z+nAccept : 0;
assert(z!=0);
while( (pcLimit)? (z<pcLimit) : (ng-- != 0) ){
char c = *z;
if( (c & 0x80) == 0 ){
if( ccm != 0L && c < 0x20 && ((1L<<c) & ccm) != 0 ) return z;
++z; /* ASCII */
}else if( (c & 0xC0) != 0xC0 ) return z; /* not a lead byte */
else{
const char *zt = z+1; /* Got lead byte, look at trail bytes.*/
do{
if( pcLimit && zt >= pcLimit ) return z;
else{
char ct = *zt++;
if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){
/* Trailing bytes are too few, too many, or invalid. */
return z;
}
}
} while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */
z = zt;
}
}
return z;
}
#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
#ifndef SQLITE_CIO_NO_TRANSLATE
# ifdef CONSIO_SPUTB
SQLITE_INTERNAL_LINKAGE int
fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
assert(pfO!=0);
# if CIO_WIN_WC_XLATE
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
if( pstReachesConsole(ppst) ){
int rv;
maybeSetupAsConsole(ppst, 1);
rv = conZstrEmit(ppst, cBuf, nAccept);
if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
return rv;
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfO);
# if CIO_WIN_WC_XLATE
}
# endif
}
# endif
SQLITE_INTERNAL_LINKAGE int
oPutbUtf8(const char *cBuf, int nAccept){
FILE *pfOut;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
# if CIO_WIN_WC_XLATE
PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
# else
getEmitStreamInfo(1, &pst, &pfOut);
# endif
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
return conZstrEmit(ppst, cBuf, nAccept);
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfOut);
# if CIO_WIN_WC_XLATE
}
# endif
}
/*
** Flush the given output stream. Return non-zero for success, else 0.
*/
#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
SQLITE_INTERNAL_LINKAGE int
fFlushBuffer(FILE *pfOut){
# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE)
return FlushFileBuffers(handleOfFile(pfOut))? 1 : 0;
# else
return fflush(pfOut);
# endif
}
#endif
#if CIO_WIN_WC_XLATE \
&& !defined(SHELL_OMIT_FIO_DUPE) \
&& defined(SQLITE_USE_ONLY_WIN32)
static struct FileAltIds {
int fd;
HANDLE fh;
} altIdsOfFile(FILE *pf){
struct FileAltIds rv = { _fileno(pf) };
union { intptr_t osfh; HANDLE fh; } fid = {
(rv.fd>=0)? _get_osfhandle(rv.fd) : (intptr_t)INVALID_HANDLE_VALUE
};
rv.fh = fid.fh;
return rv;
}
SQLITE_INTERNAL_LINKAGE size_t
cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){
size_t rv = 0;
struct FileAltIds fai = altIdsOfFile(pf);
int fmode = _setmode(fai.fd, _O_BINARY);
_setmode(fai.fd, fmode);
while( rv < ocnt ){
size_t nbo = osz;
while( nbo > 0 ){
DWORD dwno = (nbo>(1L<<24))? 1L<<24 : (DWORD)nbo;
BOOL wrc = TRUE;
BOOL genCR = (fmode & _O_TEXT)!=0;
if( genCR ){
const char *pnl = (const char*)memchr(buf, '\n', nbo);
if( pnl ) nbo = pnl - (const char*)buf;
else genCR = 0;
}
if( dwno>0 ) wrc = WriteFile(fai.fh, buf, dwno, 0,0);
if( genCR && wrc ){
wrc = WriteFile(fai.fh, "\r\n", 2, 0,0);
++dwno; /* Skip over the LF */
}
if( !wrc ) return rv;
buf = (const char*)buf + dwno;
nbo += dwno;
}
++rv;
}
return rv;
}
/* An fgets() equivalent, using Win32 file API for actual input.
** Input ends when given buffer is filled or a newline is read.
** If the FILE object is in text mode, swallows 0x0D. (ASCII CR)
*/
SQLITE_INTERNAL_LINKAGE char *
cfGets(char *cBuf, int n, FILE *pf){
int nci = 0;
struct FileAltIds fai = altIdsOfFile(pf);
int fmode = _setmode(fai.fd, _O_BINARY);
BOOL eatCR = (fmode & _O_TEXT)!=0;
_setmode(fai.fd, fmode);
while( nci < n-1 ){
DWORD nr;
if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break;
if( nr>0 && (!eatCR || cBuf[nci]!='\r') ){
nci += nr;
if( cBuf[nci-nr]=='\n' ) break;
}
}
if( nci < n ) cBuf[nci] = 0;
return (nci>0)? cBuf : 0;
}
# else
# define cfWrite(b,os,no,f) fwrite(b,os,no,f)
# define cfGets(b,n,f) fgets(b,n,f)
# endif
# ifdef CONSIO_EPUTB
SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept){
FILE *pfErr;
PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
# if CIO_WIN_WC_XLATE
if( pstReachesConsole(ppst) ){
return conZstrEmit(ppst, cBuf, nAccept);
}else {
# endif
return (int)cfWrite(cBuf, 1, nAccept, pfErr);
# if CIO_WIN_WC_XLATE
}
# endif
}
# endif /* defined(CONSIO_EPUTB) */
SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
if( pfIn==0 ) pfIn = stdin;
# if CIO_WIN_WC_XLATE
if( pfIn == consoleInfo.pstSetup[0].pf
&& (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
# if CIO_WIN_WC_XLATE==1
# define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
WCHAR wcBuf[SHELL_GULP+1];
int lend = 0, noc = 0;
if( ncMax > 0 ) cBuf[0] = 0;
while( noc < ncMax-8-1 && !lend ){
/* There is room for at least 2 more characters and a 0-terminator. */
int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
# undef SHELL_GULP
DWORD nbr = 0;
BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
/* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
DWORD nbrx;
bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
if( bRC ) nbr += nbrx;
}
if( !bRC || (noc==0 && nbr==0) ) return 0;
if( nbr > 0 ){
int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
if( nmb != 0 && noc+nmb <= ncMax ){
int iseg = noc;
nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
noc += nmb;
/* Fixup line-ends as coded by Windows for CR (or "Enter".)
** This is done without regard for any setMode{Text,Binary}()
** call that might have been done on the interactive input.
*/
if( noc > 0 ){
if( cBuf[noc-1]=='\n' ){
lend = 1;
if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
}
}
/* Check for ^Z (anywhere in line) too, to act as EOF. */
while( iseg < noc ){
if( cBuf[iseg]=='\x1a' ){
noc = iseg; /* Chop ^Z and anything following. */
lend = 1; /* Counts as end of line too. */
break;
}
++iseg;
}
}else break; /* Drop apparent garbage in. (Could assert.) */
}else break;
}
/* If got nothing, (after ^Z chop), must be at end-of-file. */
if( noc > 0 ){
cBuf[noc] = 0;
return cBuf;
}else return 0;
# endif
}else{
# endif
return cfGets(cBuf, ncMax, pfIn);
# if CIO_WIN_WC_XLATE
}
# endif
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#if defined(_MSC_VER)
# pragma warning(default : 4204)
#endif
#undef SHELL_INVALID_FILE_PTR
-287
View File
@@ -1,287 +0,0 @@
/*
** 2023 November 1
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
********************************************************************************
** This file exposes various interfaces used for console and other I/O
** by the SQLite project command-line tools. These interfaces are used
** at either source conglomeration time, compilation time, or run time.
** This source provides for either inclusion into conglomerated,
** "single-source" forms or separate compilation then linking.
**
** Platform dependencies are "hidden" here by various stratagems so
** that, provided certain conditions are met, the programs using this
** source or object code compiled from it need no explicit conditional
** compilation in their source for their console and stream I/O.
**
** The symbols and functionality exposed here are not a public API.
** This code may change in tandem with other project code as needed.
**
** When this .h file and its companion .c are directly incorporated into
** a source conglomeration (such as shell.c), the preprocessor symbol
** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
** translation for Windows is effected for the build.
*/
#define HAVE_CONSOLE_IO_H 1
#ifndef SQLITE_INTERNAL_LINKAGE
# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
# include <stdio.h>
#else
# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */
#endif
#ifndef SQLITE3_H
# include "sqlite3.h"
#endif
#ifndef SQLITE_CIO_NO_CLASSIFY
/* Define enum for use with following function. */
typedef enum StreamsAreConsole {
SAC_NoConsole = 0,
SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4,
SAC_AnyConsole = 0x7
} StreamsAreConsole;
/*
** Classify the three standard I/O streams according to whether
** they are connected to a console attached to the process.
**
** Returns the bit-wise OR of SAC_{In,Out,Err}Console values,
** or SAC_NoConsole if none of the streams reaches a console.
**
** This function should be called before any I/O is done with
** the given streams. As a side-effect, the given inputs are
** recorded so that later I/O operations on them may be done
** differently than the C library FILE* I/O would be done,
** iff the stream is used for the I/O functions that follow,
** and to support the ones that use an implicit stream.
**
** On some platforms, stream or console mode alteration (aka
** "Setup") may be made which is undone by consoleRestore().
*/
SQLITE_INTERNAL_LINKAGE StreamsAreConsole
consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr );
/* A usual call for convenience: */
#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr)
/*
** After an initial call to consoleClassifySetup(...), renew
** the same setup it effected. (A call not after is an error.)
** This will restore state altered by consoleRestore();
**
** Applications which run an inferior (child) process which
** inherits the same I/O streams may call this function after
** such a process exits to guard against console mode changes.
*/
SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void);
/*
** Undo any side-effects left by consoleClassifySetup(...).
**
** This should be called after consoleClassifySetup() and
** before the process terminates normally. It is suitable
** for use with the atexit() C library procedure. After
** this call, no console I/O should be done until one of
** console{Classify or Renew}Setup(...) is called again.
**
** Applications which run an inferior (child) process that
** inherits the same I/O streams might call this procedure
** before so that said process will have a console setup
** however users have configured it or come to expect.
*/
SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void );
#else /* defined(SQLITE_CIO_NO_CLASSIFY) */
# define consoleClassifySetup(i,o,e)
# define consoleRenewSetup()
# define consoleRestore()
#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */
#ifndef SQLITE_CIO_NO_REDIRECT
/*
** Set stream to be used for the functions below which write
** to "the designated X stream", where X is Output or Error.
** Returns the previous value.
**
** Alternatively, pass the special value, invalidFileStream,
** to get the designated stream value without setting it.
**
** Before the designated streams are set, they default to
** those passed to consoleClassifySetup(...), and before
** that is called they default to stdout and stderr.
**
** It is error to close a stream so designated, then, without
** designating another, use the corresponding {o,e}Emit(...).
*/
SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream;
SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf);
# ifdef CONSIO_SET_ERROR_STREAM
SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf);
# endif
#else
# define setOutputStream(pf)
# define setErrorStream(pf)
#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */
#ifndef SQLITE_CIO_NO_TRANSLATE
/*
** Emit output like fprintf(). If the output is going to the
** console and translation from UTF-8 is necessary, perform
** the needed translation. Otherwise, write formatted output
** to the provided stream almost as-is, possibly with newline
** translation as specified by set{Binary,Text}Mode().
*/
SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...);
/* Like fPrintfUtf8 except stream is always the designated output. */
SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...);
/* Like fPrintfUtf8 except stream is always the designated error. */
SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...);
/*
** Emit output like fputs(). If the output is going to the
** console and translation from UTF-8 is necessary, perform
** the needed translation. Otherwise, write given text to the
** provided stream almost as-is, possibly with newline
** translation as specified by set{Binary,Text}Mode().
*/
SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO);
/* Like fPutsUtf8 except stream is always the designated output. */
SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z);
/* Like fPutsUtf8 except stream is always the designated error. */
SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
/*
** Emit output like fPutsUtf8(), except that the length of the
** accepted char or character sequence is limited by nAccept.
**
** Returns the number of accepted char values.
*/
#ifdef CONSIO_SPUTB
SQLITE_INTERNAL_LINKAGE int
fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
/* Like fPutbUtf8 except stream is always the designated output. */
#endif
SQLITE_INTERNAL_LINKAGE int
oPutbUtf8(const char *cBuf, int nAccept);
/* Like fPutbUtf8 except stream is always the designated error. */
#ifdef CONSIO_EPUTB
SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept);
#endif
/*
** Flush the given output stream. Return non-zero for success, else 0.
*/
#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
SQLITE_INTERNAL_LINKAGE int
fFlushBuffer(FILE *pfOut);
#endif
/*
** Collect input like fgets(...) with special provisions for input
** from the console on such platforms as require same. Newline
** translation may be done as set by set{Binary,Text}Mode().
** As a convenience, pfIn==NULL is treated as stdin.
*/
SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
/* Like fGetsUtf8 except stream is always the designated input. */
/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#ifndef SQLITE_CIO_NO_SETMODE
/*
** Set given stream for binary mode, where newline translation is
** not done, or for text mode where, for some platforms, newlines
** are translated to the platform's conventional char sequence.
** If bFlush true, flush the stream.
**
** An additional side-effect is that if the stream is one passed
** to consoleClassifySetup() as an output, it is flushed first.
**
** Note that binary/text mode has no effect on console I/O
** translation. On all platforms, newline to the console starts
** a new line and CR,LF chars from the console become a newline.
*/
SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush);
SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush);
#endif
#ifdef SQLITE_CIO_PROMPTED_IN
typedef struct Prompts {
int numPrompts;
const char **azPrompts;
} Prompts;
/*
** Macros for use of a line editor.
**
** The following macros define operations involving use of a
** line-editing library or simple console interaction.
** A "T" argument is a text (char *) buffer or filename.
** A "N" argument is an integer.
**
** SHELL_ADD_HISTORY(T) // Record text as line(s) of history.
** SHELL_READ_HISTORY(T) // Read history from file named by T.
** SHELL_WRITE_HISTORY(T) // Write history to file named by T.
** SHELL_STIFLE_HISTORY(N) // Limit history to N entries.
**
** A console program which does interactive console input is
** expected to call:
** SHELL_READ_HISTORY(T) before collecting such input;
** SHELL_ADD_HISTORY(T) as record-worthy input is taken;
** SHELL_STIFLE_HISTORY(N) after console input ceases; then
** SHELL_WRITE_HISTORY(T) before the program exits.
*/
/*
** Retrieve a single line of input text from an input stream.
**
** If pfIn is the input stream passed to consoleClassifySetup(),
** and azPrompt is not NULL, then a prompt is issued before the
** line is collected, as selected by the isContinuation flag.
** Array azPrompt[{0,1}] holds the {main,continuation} prompt.
**
** If zBufPrior is not NULL then it is a buffer from a prior
** call to this routine that can be reused, or will be freed.
**
** The result is stored in space obtained from malloc() and
** must either be freed by the caller or else passed back to
** this function as zBufPrior for reuse.
**
** This function may call upon services of a line-editing
** library to interactively collect line edited input.
*/
SQLITE_INTERNAL_LINKAGE char *
shellGetLine(FILE *pfIn, char *zBufPrior, int nLen,
short isContinuation, Prompts azPrompt);
#endif /* defined(SQLITE_CIO_PROMPTED_IN) */
/*
** TBD: Define an interface for application(s) to generate
** completion candidates for use by the line-editor.
**
** This may be premature; the CLI is the only application
** that does this. Yet, getting line-editing melded into
** console I/O is desirable because a line-editing library
** may have to establish console operating mode, possibly
** in a way that interferes with the above functionality.
*/
#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))
/* Skip over as much z[] input char sequence as is valid UTF-8,
** limited per nAccept char's or whole characters and containing
** no char cn such that ((1<<cn) & ccm)!=0. On return, the
** sequence z:return (inclusive:exclusive) is validated UTF-8.
** Limit: nAccept>=0 => char count, nAccept<0 => character
*/
SQLITE_INTERNAL_LINKAGE const char*
zSkipValidUtf8(const char *z, int nAccept, long ccm);
#endif
+2 -55
View File
@@ -8,7 +8,6 @@
# May you share freely, never taking more than you give.
#
#***********************************************************************
# TESTRUNNER: shell
#
# The focus of this file is testing the CLI shell tool. Specifically,
# the ".recommend" command.
@@ -197,7 +196,7 @@ do_setup_rec_test $tn.9.1 {
} {
SELECT * FROM "t t" WHERE a=?
} {
CREATE INDEX "t t_idx_00000061" ON "t t"(a);
CREATE INDEX 't t_idx_00000061' ON 't t'(a);
SEARCH t t USING INDEX t t_idx_00000061 (a=?)
}
@@ -206,7 +205,7 @@ do_setup_rec_test $tn.9.2 {
} {
SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
} {
CREATE INDEX "t t_idx_00000062" ON "t t"(b);
CREATE INDEX 't t_idx_00000062' ON 't t'(b);
SEARCH t t USING INDEX t t_idx_00000062 (b>? AND b<?)
}
@@ -392,16 +391,6 @@ do_setup_rec_test $tn.18.1 {
SEARCH SomeObject USING COVERING INDEX SomeObject_idx_00000078 (x=?)
}
do_setup_rec_test $tn.19.0 {
CREATE TABLE t1("index");
} {
SELECT * FROM t1 ORDER BY "index";
} {
CREATE INDEX t1_idx_01a7214e ON t1('index');
SCAN t1 USING COVERING INDEX t1_idx_01a7214e
}
}
proc do_candidates_test {tn sql res} {
@@ -428,8 +417,6 @@ do_execsql_test 5.0 {
WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
CREATE INDEX i1 ON t1( lower(a) );
}
do_candidates_test 5.1 {
SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
@@ -459,7 +446,6 @@ do_execsql_test 5.3 {
ANALYZE;
SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
} {
t1 i1 {100 50}
t1 t1_idx_00000061 {100 50}
t1 t1_idx_00000062 {100 20}
t1 t1_idx_000123a7 {100 50 17}
@@ -468,43 +454,4 @@ do_execsql_test 5.3 {
t2 t2_idx_0001295b {100 20 5}
}
do_catchsql_test 5.4 {
SELECT sqlite_expert_rem(123, 123);
} {1 {no such function: sqlite_expert_rem}}
do_catchsql_test 5.5 {
SELECT sqlite_expert_sample();
} {1 {no such function: sqlite_expert_sample}}
if 0 {
do_test expert1-6.0 {
catchcmd :memory: {
.expert
select base64('');
.expert
select name from pragma_collation_list order by name collate uint;
}
} {0 {(no new indexes)
SCAN CONSTANT ROW
(no new indexes)
SCAN pragma_collation_list VIRTUAL TABLE INDEX 0:
USE TEMP B-TREE FOR ORDER BY
}}
}
do_execsql_test 6.0 {
CREATE TABLE x1(a, b, c, d);
CREATE INDEX x1ab ON x1(a, lower(b));
CREATE INDEX x1dcba ON x1(d, b+c, a);
}
do_candidates_test 6.1 {
SELECT * FROM x1 WHERE b=? ORDER BY a;
} {
CREATE INDEX x1_idx_0001267f ON x1(b, a);
CREATE INDEX x1_idx_00000062 ON x1(b);
}
finish_test
+32 -225
View File
@@ -15,24 +15,7 @@
#include <string.h>
#include <stdio.h>
#if !defined(SQLITE_AMALGAMATION)
#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
#endif
#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
# define ALWAYS(X) (1)
# define NEVER(X) (0)
#elif !defined(NDEBUG)
# define ALWAYS(X) ((X)?1:(assert(0),0))
# define NEVER(X) ((X)?(assert(0),1):0)
#else
# define ALWAYS(X) (X)
# define NEVER(X) (X)
#endif
#endif /* !defined(SQLITE_AMALGAMATION) */
#ifndef SQLITE_OMIT_VIRTUALTABLE
#ifndef SQLITE_OMIT_VIRTUALTABLE
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
@@ -626,7 +609,7 @@ static int expertFilter(
pCsr->pData = 0;
if( rc==SQLITE_OK ){
rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
"SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName
"SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
);
}
@@ -662,7 +645,6 @@ static int idxRegisterVtab(sqlite3expert *p){
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0, /* xIntegrity */
};
return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -698,25 +680,17 @@ static int idxGetTableInfo(
){
sqlite3_stmt *p1 = 0;
int nCol = 0;
int nTab;
int nByte;
int nTab = STRLEN(zTab);
int nByte = sizeof(IdxTable) + nTab + 1;
IdxTable *pNew = 0;
int rc, rc2;
char *pCsr = 0;
int nPk = 0;
*ppOut = 0;
if( zTab==0 ) return SQLITE_ERROR;
nTab = STRLEN(zTab);
nByte = sizeof(IdxTable) + nTab + 1;
rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
const char *zCol = (const char*)sqlite3_column_text(p1, 1);
const char *zColSeq = 0;
if( zCol==0 ){
rc = SQLITE_ERROR;
break;
}
nByte += 1 + STRLEN(zCol);
rc = sqlite3_table_column_metadata(
db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0
@@ -743,9 +717,7 @@ static int idxGetTableInfo(
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
const char *zCol = (const char*)sqlite3_column_text(p1, 1);
const char *zColSeq = 0;
int nCopy;
if( zCol==0 ) continue;
nCopy = STRLEN(zCol) + 1;
int nCopy = STRLEN(zCol) + 1;
pNew->aCol[nCol].zName = pCsr;
pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1);
memcpy(pCsr, zCol, nCopy);
@@ -769,9 +741,9 @@ static int idxGetTableInfo(
if( rc!=SQLITE_OK ){
sqlite3_free(pNew);
pNew = 0;
}else if( ALWAYS(pNew!=0) ){
}else{
pNew->zName = pCsr;
if( ALWAYS(pNew->zName!=0) ) memcpy(pNew->zName, zTab, nTab+1);
memcpy(pNew->zName, zTab, nTab+1);
}
*ppOut = pNew;
@@ -821,10 +793,6 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
*/
static int idxIdentifierRequiresQuotes(const char *zId){
int i;
int nId = STRLEN(zId);
if( sqlite3_keyword_check(zId, nId) ) return 1;
for(i=0; zId[i]; i++){
if( !(zId[i]=='_')
&& !(zId[i]>='0' && zId[i]<='9')
@@ -901,7 +869,6 @@ static int idxFindCompatible(
IdxConstraint *pT = pTail;
sqlite3_stmt *pInfo = 0;
const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
if( zIdx==0 ) continue;
/* Zero the IdxConstraint.bFlag values in the pEq list */
for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
@@ -947,19 +914,6 @@ static int idxFindCompatible(
return 0;
}
/* Callback for sqlite3_exec() with query with leading count(*) column.
* The first argument is expected to be an int*, referent to be incremented
* if that leading column is not exactly '0'.
*/
static int countNonzeros(void* pCount, int nc,
char* azResults[], char* azColumns[]){
(void)azColumns; /* Suppress unused parameter warning */
if( nc>0 && (azResults[0][0]!='0' || azResults[0][1]!=0) ){
*((int *)pCount) += 1;
}
return 0;
}
static int idxCreateFromCons(
sqlite3expert *p,
IdxScan *pScan,
@@ -986,40 +940,17 @@ static int idxCreateFromCons(
if( rc==SQLITE_OK ){
/* Hash the list of columns to come up with a name for the index */
const char *zTable = pScan->pTab->zName;
int quoteTable = idxIdentifierRequiresQuotes(zTable);
char *zName = 0; /* Index name */
int collisions = 0;
do{
int i;
char *zFind;
for(i=0; zCols[i]; i++){
h += ((h<<3) + zCols[i]);
}
sqlite3_free(zName);
zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
if( zName==0 ) break;
/* Is is unique among table, view and index names? */
zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q"
" AND type in ('index','table','view')";
zFind = sqlite3_mprintf(zFmt, zName);
i = 0;
rc = sqlite3_exec(dbm, zFind, countNonzeros, &i, 0);
assert(rc==SQLITE_OK);
sqlite3_free(zFind);
if( i==0 ){
collisions = 0;
break;
}
++collisions;
}while( collisions<50 && zName!=0 );
if( collisions ){
/* This return means "Gave up trying to find a unique index name." */
rc = SQLITE_BUSY_TIMEOUT;
}else if( zName==0 ){
char *zName; /* Index name */
int i;
for(i=0; zCols[i]; i++){
h += ((h<<3) + zCols[i]);
}
zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
if( zName==0 ){
rc = SQLITE_NOMEM;
}else{
if( quoteTable ){
zFmt = "CREATE INDEX \"%w\" ON \"%w\"(%s)";
if( idxIdentifierRequiresQuotes(zTable) ){
zFmt = "CREATE INDEX '%q' ON %Q(%s)";
}else{
zFmt = "CREATE INDEX %s ON %s(%s)";
}
@@ -1028,11 +959,7 @@ static int idxCreateFromCons(
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
if( rc!=SQLITE_OK ){
rc = SQLITE_BUSY_TIMEOUT;
}else{
idxHashAdd(&rc, &p->hIdx, zName, zIdx);
}
idxHashAdd(&rc, &p->hIdx, zName, zIdx);
}
sqlite3_free(zName);
sqlite3_free(zIdx);
@@ -1184,7 +1111,7 @@ static void idxWriteFree(IdxWrite *pTab){
** runs all the queries to see which indexes they prefer, and populates
** IdxStatement.zIdx and IdxStatement.zEQP with the results.
*/
static int idxFindIndexes(
int idxFindIndexes(
sqlite3expert *p,
char **pzErr /* OUT: Error message (sqlite3_malloc) */
){
@@ -1313,7 +1240,6 @@ static int idxProcessOneTrigger(
rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
if( zCreate==0 ) continue;
rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
}
idxFinalize(&rc, pSelect);
@@ -1416,9 +1342,8 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
if( zType==0 || zName==0 ) continue;
if( zType[0]=='v' || zType[1]=='r' ){
if( zSql ) rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
}else{
IdxTable *pTab;
rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
@@ -1500,7 +1425,7 @@ struct IdxRemCtx {
};
/*
** Implementation of scalar function sqlite_expert_rem().
** Implementation of scalar function rem().
*/
static void idxRemFunc(
sqlite3_context *pCtx,
@@ -1513,7 +1438,7 @@ static void idxRemFunc(
assert( argc==2 );
iSlot = sqlite3_value_int(argv[0]);
assert( iSlot<p->nSlot );
assert( iSlot<=p->nSlot );
pSlot = &p->aSlot[iSlot];
switch( pSlot->eType ){
@@ -1555,7 +1480,6 @@ static void idxRemFunc(
case SQLITE_BLOB:
case SQLITE_TEXT: {
int nByte = sqlite3_value_bytes(argv[1]);
const void *pData = 0;
if( nByte>pSlot->nByte ){
char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
if( zNew==0 ){
@@ -1567,11 +1491,9 @@ static void idxRemFunc(
}
pSlot->n = nByte;
if( pSlot->eType==SQLITE_BLOB ){
pData = sqlite3_value_blob(argv[1]);
if( pData ) memcpy(pSlot->z, pData, nByte);
memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
}else{
pData = sqlite3_value_text(argv[1]);
memcpy(pSlot->z, pData, nByte);
memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
}
break;
}
@@ -1623,15 +1545,8 @@ static int idxPopulateOneStat1(
const char *zComma = zCols==0 ? "" : ", ";
const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
if( zName==0 ){
/* This index contains an expression. Ignore it. */
sqlite3_free(zCols);
sqlite3_free(zOrder);
return sqlite3_reset(pIndexXInfo);
}
zCols = idxAppendText(&rc, zCols,
"%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s",
zComma, zName, nCol, zName, zColl
"%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
);
zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
}
@@ -1764,13 +1679,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
if( rc==SQLITE_OK ){
sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
rc = sqlite3_create_function(dbrem, "sqlite_expert_rem",
2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
rc = sqlite3_create_function(
dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(p->db, "sqlite_expert_sample",
0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
rc = sqlite3_create_function(
p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
);
}
@@ -1789,7 +1704,6 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
if( zTab==0 || zIdx==0 ) continue;
if( p->iSample<100 && iPrev!=iRowid ){
samplectx.target = (double)p->iSample / 100.0;
samplectx.iTarget = p->iSample;
@@ -1822,95 +1736,10 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0);
}
sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0);
sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0);
sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
return rc;
}
/*
** Define and possibly pretend to use a useless collation sequence.
** This pretense allows expert to accept SQL using custom collations.
*/
int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){
(void)up1;
(void)up2;
(void)up3;
(void)up4;
(void)up5;
assert(0); /* VDBE should never be run. */
return 0;
}
/* And a callback to register above upon actual need */
void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){
(void)up1;
sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0);
}
#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
&& !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
/*
** dummy functions for no-op implementation of UDFs during expert's work
*/
void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){
(void)up1;
(void)up2;
(void)up3;
assert(0); /* VDBE should never be run. */
}
void dummyUDFvalue(sqlite3_context *up1){
(void)up1;
assert(0); /* VDBE should never be run. */
}
/*
** Register UDFs from user database with another.
*/
int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){
sqlite3_stmt *pStmt;
int rc = sqlite3_prepare_v2(dbSrc,
"SELECT name,type,enc,narg,flags "
"FROM pragma_function_list() "
"WHERE builtin==0", -1, &pStmt, 0);
if( rc==SQLITE_OK ){
while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
int nargs = sqlite3_column_int(pStmt,3);
int flags = sqlite3_column_int(pStmt,4);
const char *name = (char*)sqlite3_column_text(pStmt,0);
const char *type = (char*)sqlite3_column_text(pStmt,1);
const char *enc = (char*)sqlite3_column_text(pStmt,2);
if( name==0 || type==0 || enc==0 ){
/* no-op. Only happens on OOM */
}else{
int ienc = SQLITE_UTF8;
int rcf = SQLITE_ERROR;
if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE;
else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE;
ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY));
if( strcmp(type,"w")==0 ){
rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0,
dummyUDF,dummyUDFvalue,0,0,0);
}else if( strcmp(type,"a")==0 ){
rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
0,dummyUDF,dummyUDFvalue);
}else if( strcmp(type,"s")==0 ){
rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
dummyUDF,0,0);
}
if( rcf!=SQLITE_OK ){
rc = rcf;
break;
}
}
}
sqlite3_finalize(pStmt);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
return rc;
}
#endif
/*
** Allocate a new sqlite3expert object.
*/
@@ -1937,32 +1766,18 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
}
}
/* Allow custom collations to be dealt with through prepare. */
if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS);
if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS);
#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
&& !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
/* Register UDFs from database [db] with [dbm] and [dbv]. */
if( rc==SQLITE_OK ){
rc = registerUDFs(pNew->db, pNew->dbm);
}
if( rc==SQLITE_OK ){
rc = registerUDFs(pNew->db, pNew->dbv);
}
#endif
/* Copy the entire schema of database [db] into [dbm]. */
if( rc==SQLITE_OK ){
sqlite3_stmt *pSql = 0;
sqlite3_stmt *pSql;
rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg,
"SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'"
" AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid"
" AND sql NOT LIKE 'CREATE VIRTUAL %%'"
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
if( zSql ) rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
}
idxFinalize(&rc, pSql);
}
@@ -2027,10 +1842,6 @@ int sqlite3_expert_sql(
while( rc==SQLITE_OK && zStmt && zStmt[0] ){
sqlite3_stmt *pStmt = 0;
/* Ensure that the provided statement compiles against user's DB. */
rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt);
if( rc!=SQLITE_OK ) break;
sqlite3_finalize(pStmt);
rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
if( rc==SQLITE_OK ){
if( pStmt ){
@@ -2072,10 +1883,6 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
/* Create candidate indexes within the in-memory database file */
if( rc==SQLITE_OK ){
rc = idxCreateCandidates(p);
}else if ( rc==SQLITE_BUSY_TIMEOUT ){
if( pzErr )
*pzErr = sqlite3_mprintf("Cannot find a unique index name to propose.");
return rc;
}
/* Generate the stat1 data */
+9 -1
View File
@@ -16,7 +16,15 @@
#include "sqlite3expert.h"
#include <assert.h>
#include <string.h>
#include "tclsqlite.h"
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#ifndef SQLITE_OMIT_VIRTUALTABLE
+2
View File
@@ -0,0 +1,2 @@
This folder contains source code to the first full-text search
extension for SQLite.
+404
View File
@@ -0,0 +1,404 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "ft_hash.h"
void *malloc_and_zero(int n){
void *p = malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants HASH_INT, HASH_POINTER,
** HASH_BINARY, or HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer. CopyKey only makes
** sense for HASH_STRING and HASH_BINARY and is ignored
** for other key classes.
*/
void HashInit(Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=HASH_STRING && keyClass<=HASH_BINARY );
pNew->keyClass = keyClass;
#if 0
if( keyClass==HASH_POINTER || keyClass==HASH_INT ) copyKey = 0;
#endif
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
pNew->xMalloc = malloc_and_zero;
pNew->xFree = free;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void HashClear(Hash *pH){
HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree(elem);
elem = next_elem;
}
pH->count = 0;
}
#if 0 /* NOT USED */
/*
** Hash and comparison functions when the mode is HASH_INT
*/
static int intHash(const void *pKey, int nKey){
return nKey ^ (nKey<<8) ^ (nKey>>8);
}
static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
return n2 - n1;
}
#endif
#if 0 /* NOT USED */
/*
** Hash and comparison functions when the mode is HASH_POINTER
*/
static int ptrHash(const void *pKey, int nKey){
uptr x = Addr(pKey);
return x ^ (x<<8) ^ (x>>8);
}
static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( pKey1==pKey2 ) return 0;
if( pKey1<pKey2 ) return -1;
return 1;
}
#endif
/*
** Hash and comparison functions when the mode is HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
#if 0 /* HASH_INT and HASH_POINTER are never used */
switch( keyClass ){
case HASH_INT: return &intHash;
case HASH_POINTER: return &ptrHash;
case HASH_STRING: return &strHash;
case HASH_BINARY: return &binHash;;
default: break;
}
return 0;
#else
if( keyClass==HASH_STRING ){
return &strHash;
}else{
assert( keyClass==HASH_BINARY );
return &binHash;
}
#endif
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
#if 0 /* HASH_INT and HASH_POINTER are never used */
switch( keyClass ){
case HASH_INT: return &intCompare;
case HASH_POINTER: return &ptrCompare;
case HASH_STRING: return &strCompare;
case HASH_BINARY: return &binCompare;
default: break;
}
return 0;
#else
if( keyClass==HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==HASH_BINARY );
return &binCompare;
}
#endif
}
/* Link an element into the hash table
*/
static void insertElement(
Hash *pH, /* The complete hash table */
struct _ht *pEntry, /* The entry into which pNew is inserted */
HashElem *pNew /* The element to be inserted */
){
HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(Hash *pH, int new_size){
struct _ht *new_ht; /* The new hash table */
HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
if( new_ht==0 ) return;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static HashElem *findElementGivenHash(
const Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
Hash *pH, /* The pH containing "elem" */
HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *HashFind(const Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
HashElem *elem; /* Used to loop thru the element list */
HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = pH->xMalloc( nKey );
if( new_elem->pKey==0 ){
pH->xFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
pH->xFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
+111
View File
@@ -0,0 +1,111 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implementation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _HASH_H_
#define _HASH_H_
/* Forward declarations of structures. */
typedef struct Hash Hash;
typedef struct HashElem HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
HashElem *first; /* The first element of the array */
void *(*xMalloc)(int); /* malloc() function to use */
void (*xFree)(void *); /* free() function to use */
int htsize; /* Number of buckets in the hash table */
struct _ht { /* the hash table */
int count; /* Number of entries with this hash */
HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct HashElem {
HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 4 different modes of operation for a hash table:
**
** HASH_INT nKey is used as the key and pKey is ignored.
**
** HASH_POINTER pKey is used as the key and nKey is ignored.
**
** HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made for HASH_STRING and HASH_BINARY
** if the copyKey parameter to HashInit is 1.
*/
/* #define HASH_INT 1 // NOT USED */
/* #define HASH_POINTER 2 // NOT USED */
#define HASH_STRING 3
#define HASH_BINARY 4
/*
** Access routines. To delete, insert a NULL pointer.
*/
void HashInit(Hash*, int keytype, int copyKey);
void *HashInsert(Hash*, const void *pKey, int nKey, void *pData);
void *HashFind(const Hash*, const void *pKey, int nKey);
void HashClear(Hash*);
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** Hash h;
** HashElem *p;
** ...
** for(p=HashFirst(&h); p; p=HashNext(p)){
** SomeStructure *pData = HashData(p);
** // do something with pData
** }
*/
#define HashFirst(H) ((H)->first)
#define HashNext(E) ((E)->next)
#define HashData(E) ((E)->data)
#define HashKey(E) ((E)->pKey)
#define HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define HashCount(H) ((H)->count)
#endif /* _HASH_H_ */
+3348
View File
File diff suppressed because it is too large Load Diff
+11
View File
@@ -0,0 +1,11 @@
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3Fts1Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
+369
View File
@@ -0,0 +1,369 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include "fts1_hash.h"
static void *malloc_and_zero(int n){
void *p = malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants
** FTS1_HASH_BINARY or FTS1_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
void sqlite3Fts1HashInit(fts1Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS1_HASH_STRING && keyClass<=FTS1_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
pNew->xMalloc = malloc_and_zero;
pNew->xFree = free;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void sqlite3Fts1HashClear(fts1Hash *pH){
fts1HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
fts1HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree(elem);
elem = next_elem;
}
pH->count = 0;
}
/*
** Hash and comparison functions when the mode is FTS1_HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is FTS1_HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
if( keyClass==FTS1_HASH_STRING ){
return &strHash;
}else{
assert( keyClass==FTS1_HASH_BINARY );
return &binHash;
}
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
if( keyClass==FTS1_HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==FTS1_HASH_BINARY );
return &binCompare;
}
}
/* Link an element into the hash table
*/
static void insertElement(
fts1Hash *pH, /* The complete hash table */
struct _fts1ht *pEntry, /* The entry into which pNew is inserted */
fts1HashElem *pNew /* The element to be inserted */
){
fts1HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(fts1Hash *pH, int new_size){
struct _fts1ht *new_ht; /* The new hash table */
fts1HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts1ht *)pH->xMalloc( new_size*sizeof(struct _fts1ht) );
if( new_ht==0 ) return;
if( pH->ht ) pH->xFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static fts1HashElem *findElementGivenHash(
const fts1Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
fts1HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts1ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
fts1Hash *pH, /* The pH containing "elem" */
fts1HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts1ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
pH->xFree(elem->pKey);
}
pH->xFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
fts1HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *sqlite3Fts1HashFind(const fts1Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
fts1HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts1HashInsert(
fts1Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
fts1HashElem *elem; /* Used to loop thru the element list */
fts1HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (fts1HashElem*)pH->xMalloc( sizeof(fts1HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = pH->xMalloc( nKey );
if( new_elem->pKey==0 ){
pH->xFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
pH->xFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
+112
View File
@@ -0,0 +1,112 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implementation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _FTS1_HASH_H_
#define _FTS1_HASH_H_
/* Forward declarations of structures. */
typedef struct fts1Hash fts1Hash;
typedef struct fts1HashElem fts1HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct fts1Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
fts1HashElem *first; /* The first element of the array */
void *(*xMalloc)(int); /* malloc() function to use */
void (*xFree)(void *); /* free() function to use */
int htsize; /* Number of buckets in the hash table */
struct _fts1ht { /* the hash table */
int count; /* Number of entries with this hash */
fts1HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct fts1HashElem {
fts1HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
** FTS1_HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** FTS1_HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made if the copyKey parameter to fts1HashInit is 1.
*/
#define FTS1_HASH_STRING 1
#define FTS1_HASH_BINARY 2
/*
** Access routines. To delete, insert a NULL pointer.
*/
void sqlite3Fts1HashInit(fts1Hash*, int keytype, int copyKey);
void *sqlite3Fts1HashInsert(fts1Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts1HashFind(const fts1Hash*, const void *pKey, int nKey);
void sqlite3Fts1HashClear(fts1Hash*);
/*
** Shorthand for the functions above
*/
#define fts1HashInit sqlite3Fts1HashInit
#define fts1HashInsert sqlite3Fts1HashInsert
#define fts1HashFind sqlite3Fts1HashFind
#define fts1HashClear sqlite3Fts1HashClear
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** fts1Hash h;
** fts1HashElem *p;
** ...
** for(p=fts1HashFirst(&h); p; p=fts1HashNext(p)){
** SomeStructure *pData = fts1HashData(p);
** // do something with pData
** }
*/
#define fts1HashFirst(H) ((H)->first)
#define fts1HashNext(E) ((E)->next)
#define fts1HashData(E) ((E)->data)
#define fts1HashKey(E) ((E)->pKey)
#define fts1HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define fts1HashCount(H) ((H)->count)
#endif /* _FTS1_HASH_H_ */
+643
View File
@@ -0,0 +1,643 @@
/*
** 2006 September 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "fts1_tokenizer.h"
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
sqlite3_tokenizer base; /* Base class */
} porter_tokenizer;
/*
** Class derived from sqlit3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *zInput; /* input we are tokenizing */
int nInput; /* size of the input */
int iOffset; /* current position in zInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nAllocated; /* space allocated to zToken buffer */
} porter_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module porterTokenizerModule;
/*
** Create a new tokenizer instance.
*/
static int porterCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
porter_tokenizer *t;
t = (porter_tokenizer *) calloc(sizeof(*t), 1);
if( t==NULL ) return SQLITE_NOMEM;
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int porterDestroy(sqlite3_tokenizer *pTokenizer){
free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is zInput[0..nInput-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int porterOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, int nInput, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
porter_tokenizer_cursor *c;
c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->zInput = zInput;
if( zInput==0 ){
c->nInput = 0;
}else if( nInput<0 ){
c->nInput = (int)strlen(zInput);
}else{
c->nInput = nInput;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** porterOpen() above.
*/
static int porterClose(sqlite3_tokenizer_cursor *pCursor){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
free(c->zToken);
free(c);
return SQLITE_OK;
}
/*
** Vowel or consonant
*/
static const char cType[] = {
0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
1, 1, 1, 2, 1
};
/*
** isConsonant() and isVowel() determine if their first character in
** the string they point to is a consonant or a vowel, according
** to Porter ruls.
**
** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
** 'Y' is a consonant unless it follows another consonant,
** in which case it is a vowel.
**
** In these routine, the letters are in reverse order. So the 'y' rule
** is that 'y' is a consonant unless it is followed by another
** consonent.
*/
static int isVowel(const char*);
static int isConsonant(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return j;
return z[1]==0 || isVowel(z + 1);
}
static int isVowel(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return 1-j;
return isConsonant(z + 1);
}
/*
** Let any sequence of one or more vowels be represented by V and let
** C be sequence of one or more consonants. Then every word can be
** represented as:
**
** [C] (VC){m} [V]
**
** In prose: A word is an optional consonant followed by zero or
** vowel-consonant pairs followed by an optional vowel. "m" is the
** number of vowel consonant pairs. This routine computes the value
** of m for the first i bytes of a word.
**
** Return true if the m-value for z is 1 or more. In other words,
** return true if z contains at least one vowel that is followed
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/* Like mgt0 above except we are looking for a value of m which is
** exactly 1
*/
static int m_eq_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 1;
while( isConsonant(z) ){ z++; }
return *z==0;
}
/* Like mgt0 above except we are looking for a value of m>1 instead
** or m>0
*/
static int m_gt_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if there is a vowel anywhere within z[0..n-1]
*/
static int hasVowel(const char *z){
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if the word ends in a double consonant.
**
** The text is reversed here. So we are really looking at
** the first two characters of z[].
*/
static int doubleConsonant(const char *z){
return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
}
/*
** Return TRUE if the word ends with three letters which
** are consonant-vowel-consonent and where the final consonant
** is not 'w', 'x', or 'y'.
**
** The word is reversed here. So we are really checking the
** first three letters and the first one cannot be in [wxy].
*/
static int star_oh(const char *z){
return
z[0]!=0 && isConsonant(z) &&
z[0]!='w' && z[0]!='x' && z[0]!='y' &&
z[1]!=0 && isVowel(z+1) &&
z[2]!=0 && isConsonant(z+2);
}
/*
** If the word ends with zFrom and xCond() is true for the stem
** of the word that preceeds the zFrom ending, then change the
** ending to zTo.
**
** The input word *pz and zFrom are both in reverse order. zTo
** is in normal order.
**
** Return TRUE if zFrom matches. Return FALSE if zFrom does not
** match. Not that TRUE is returned even if xCond() fails and
** no substitution occurs.
*/
static int stem(
char **pz, /* The word being stemmed (Reversed) */
const char *zFrom, /* If the ending matches this... (Reversed) */
const char *zTo, /* ... change the ending to this (not reversed) */
int (*xCond)(const char*) /* Condition that must be true */
){
char *z = *pz;
while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
if( *zFrom!=0 ) return 0;
if( xCond && !xCond(z) ) return 1;
while( *zTo ){
*(--z) = *(zTo++);
}
*pz = z;
return 1;
}
/*
** This is the fallback stemmer used when the porter stemmer is
** inappropriate. The input word is copied into the output with
** US-ASCII case folding. If the input word is too long (more
** than 20 bytes if it contains no digits or more than 6 bytes if
** it contains digits) then word is truncated to 20 or 6 bytes
** by taking 10 or 3 bytes from the beginning and end.
*/
static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, mx, j;
int hasDigit = 0;
for(i=0; i<nIn; i++){
int c = zIn[i];
if( c>='A' && c<='Z' ){
zOut[i] = c - 'A' + 'a';
}else{
if( c>='0' && c<='9' ) hasDigit = 1;
zOut[i] = c;
}
}
mx = hasDigit ? 3 : 10;
if( nIn>mx*2 ){
for(j=mx, i=nIn-mx; i<nIn; i++, j++){
zOut[j] = zOut[i];
}
i = j;
}
zOut[i] = 0;
*pnOut = i;
}
/*
** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
** zOut is at least big enough to hold nIn bytes. Write the actual
** size of the output word (exclusive of the '\0' terminator) into *pnOut.
**
** Any upper-case characters in the US-ASCII character set ([A-Z])
** are converted to lower case. Upper-case UTF characters are
** unchanged.
**
** Words that are longer than about 20 bytes are stemmed by retaining
** a few bytes from the beginning and the end of the word. If the
** word contains digits, 3 bytes are taken from the beginning and
** 3 bytes from the end. For long words without digits, 10 bytes
** are taken from each end. US-ASCII case folding still applies.
**
** If the input word contains not digits but does characters not
** in [a-zA-Z] then no stemming is attempted and this routine just
** copies the input into the input into the output with US-ASCII
** case folding.
**
** Stemming never increases the length of the word. So there is
** no chance of overflowing the zOut buffer.
*/
static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, j, c;
char zReverse[28];
char *z, *z2;
if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
c = zIn[i];
if( c>='A' && c<='Z' ){
zReverse[j] = c + 'a' - 'A';
}else if( c>='a' && c<='z' ){
zReverse[j] = c;
}else{
/* The use of a character not in [a-zA-Z] means that we fallback
** to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
}
memset(&zReverse[sizeof(zReverse)-5], 0, 5);
z = &zReverse[j+1];
/* Step 1a */
if( z[0]=='s' ){
if(
!stem(&z, "sess", "ss", 0) &&
!stem(&z, "sei", "i", 0) &&
!stem(&z, "ss", "ss", 0)
){
z++;
}
}
/* Step 1b */
z2 = z;
if( stem(&z, "dee", "ee", m_gt_0) ){
/* Do nothing. The work was all in the test */
}else if(
(stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
&& z!=z2
){
if( stem(&z, "ta", "ate", 0) ||
stem(&z, "lb", "ble", 0) ||
stem(&z, "zi", "ize", 0) ){
/* Do nothing. The work was all in the test */
}else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
z++;
}else if( m_eq_1(z) && star_oh(z) ){
*(--z) = 'e';
}
}
/* Step 1c */
if( z[0]=='y' && hasVowel(z+1) ){
z[0] = 'i';
}
/* Step 2 */
switch( z[1] ){
case 'a':
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
break;
case 'g':
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
break;
}
/* Step 4 */
switch( z[1] ){
case 'a':
if( z[0]=='l' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'c':
if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'e':
if( z[0]=='r' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'i':
if( z[0]=='c' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'l':
if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'n':
if( z[0]=='t' ){
if( z[2]=='a' ){
if( m_gt_1(z+3) ){
z += 3;
}
}else if( z[2]=='e' ){
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
case 'o':
if( z[0]=='u' ){
if( m_gt_1(z+2) ){
z += 2;
}
}else if( z[3]=='s' || z[3]=='t' ){
stem(&z, "noi", "", m_gt_1);
}
break;
case 's':
if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
case 't':
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
z += 3;
}
break;
case 'v':
case 'z':
if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
}
/* Step 5a */
if( z[0]=='e' ){
if( m_gt_1(z+1) ){
z++;
}else if( m_eq_1(z+1) && !star_oh(z+1) ){
z++;
}
}
/* Step 5b */
if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
z++;
}
/* z[] is now the stemmed word in reverse order. Flip it back
** around into forward order and return.
*/
*pnOut = i = strlen(z);
zOut[i] = 0;
while( *z ){
zOut[--i] = *(z++);
}
}
/*
** Characters that can be part of a token. We assume any character
** whose value is greater than 0x80 (any UTF character) can be
** part of a token. In other words, delimiters all must have
** values of 0x7f or lower.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
#define idChar(C) (((ch=C)&0x80)!=0 || (ch>0x2f && isIdChar[ch-0x30]))
#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !isIdChar[ch-0x30]))
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to porterOpen().
*/
static int porterNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
const char **pzToken, /* OUT: *pzToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
const char *z = c->zInput;
while( c->iOffset<c->nInput ){
int iStartOffset, ch;
/* Scan past delimiter characters */
while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int n = c->iOffset-iStartOffset;
if( n>c->nAllocated ){
c->nAllocated = n+20;
c->zToken = realloc(c->zToken, c->nAllocated);
if( c->zToken==NULL ) return SQLITE_NOMEM;
}
porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
*pzToken = c->zToken;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the porter-stemmer tokenizer
*/
static const sqlite3_tokenizer_module porterTokenizerModule = {
0,
porterCreate,
porterDestroy,
porterOpen,
porterClose,
porterNext,
};
/*
** Allocate a new porter tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts1PorterTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &porterTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
+90
View File
@@ -0,0 +1,90 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _FTS1_TOKENIZER_H_
#define _FTS1_TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface.
*/
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
struct sqlite3_tokenizer_module {
int iVersion; /* currently 0 */
/*
** Create and destroy a tokenizer. argc/argv are passed down from
** the fulltext virtual table creation to allow customization.
*/
int (*xCreate)(int argc, const char *const*argv,
sqlite3_tokenizer **ppTokenizer);
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Tokenize a particular input. Call xOpen() to prepare to
** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
** xClose() to free any internal state. The pInput passed to
** xOpen() must exist until the cursor is closed. The ppToken
** result from xNext() is only valid until the next call to xNext()
** or until xClose() is called.
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xOpen)(sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor);
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition);
};
struct sqlite3_tokenizer {
const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
/*
** Get the module for a tokenizer which generates tokens based on a
** set of non-token characters. The default is to break tokens at any
** non-alnum character, though the set of delimiters can also be
** specified by the first argv argument to xCreate().
*/
/* TODO(shess) This doesn't belong here. Need some sort of
** registration process.
*/
void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif /* _FTS1_TOKENIZER_H_ */
+221
View File
@@ -0,0 +1,221 @@
/*
** The author disclaims copyright to this source code.
**
*************************************************************************
** Implementation of the "simple" full-text-search tokenizer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS1 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS1 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "fts1_tokenizer.h"
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
char delim[128]; /* flag ASCII delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
int iOffset; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *pToken; /* storage for current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module simpleTokenizerModule;
static int isDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) calloc(sizeof(*t), 1);
if( t==NULL ) return SQLITE_NOMEM;
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
int i, n = strlen(argv[1]);
for(i=0; i<n; i++){
unsigned char ch = argv[1][i];
/* We explicitly don't support UTF-8 delimiters for now. */
if( ch>=0x80 ){
free(t);
return SQLITE_ERROR;
}
t->delim[ch] = 1;
}
} else {
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
t->delim[i] = !isalnum(i);
}
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int simpleOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *pInput, int nBytes, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->pInput = pInput;
if( pInput==0 ){
c->nBytes = 0;
}else if( nBytes<0 ){
c->nBytes = (int)strlen(pInput);
}else{
c->nBytes = nBytes;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->pToken = NULL; /* no space allocated, yet. */
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
free(c->pToken);
free(c);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
unsigned char *p = (unsigned char *)c->pInput;
while( c->iOffset<c->nBytes ){
int iStartOffset;
/* Scan past delimiter characters */
while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int i, n = c->iOffset-iStartOffset;
if( n>c->nTokenAllocated ){
c->nTokenAllocated = n+20;
c->pToken = realloc(c->pToken, c->nTokenAllocated);
if( c->pToken==NULL ) return SQLITE_NOMEM;
}
for(i=0; i<n; i++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
/*
** Allocate a new simple tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts1SimpleTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &simpleTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
+1511
View File
File diff suppressed because it is too large Load Diff
+11
View File
@@ -0,0 +1,11 @@
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int fulltext_init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
+174
View File
@@ -0,0 +1,174 @@
/*
** The author disclaims copyright to this source code.
**
*************************************************************************
** Implementation of the "simple" full-text-search tokenizer.
*/
#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include "tokenizer.h"
/* Duplicate a string; the caller must free() the returned string.
* (We don't use strdup() since it's not part of the standard C library and
* may not be available everywhere.) */
/* TODO(shess) Copied from fulltext.c, consider util.c for such
** things. */
static char *string_dup(const char *s){
char *str = malloc(strlen(s) + 1);
strcpy(str, s);
return str;
}
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
const char *zDelim; /* token delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
const char *pCurrent; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nTokenBytes; /* actual size of current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
static int simpleCreate(
int argc, const char **argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
t->zDelim = string_dup(argv[1]);
} else {
/* Build a string excluding alphanumeric ASCII characters */
char zDelim[0x80]; /* nul-terminated, so nul not a member */
int i, j;
for(i=1, j=0; i<0x80; i++){
if( !isalnum(i) ){
zDelim[j++] = i;
}
}
zDelim[j++] = '\0';
assert( j<=sizeof(zDelim) );
t->zDelim = string_dup(zDelim);
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
free((void *) t->zDelim);
free(t);
return SQLITE_OK;
}
static int simpleOpen(
sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
c->pInput = pInput;
c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
c->pCurrent = c->pInput; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nTokenBytes = 0;
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
if( NULL!=c->zToken ){
free(c->zToken);
}
free(c);
return SQLITE_OK;
}
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
int ii;
while( c->pCurrent-c->pInput<c->nBytes ){
int n = (int) strcspn(c->pCurrent, t->zDelim);
if( n>0 ){
if( n+1>c->nTokenAllocated ){
c->zToken = realloc(c->zToken, n+1);
}
for(ii=0; ii<n; ii++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
char ch = c->pCurrent[ii];
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
}
c->zToken[n] = '\0';
*ppToken = c->zToken;
*pnBytes = n;
*piStartOffset = (int) (c->pCurrent-c->pInput);
*piEndOffset = *piStartOffset+n;
*piPosition = c->iToken++;
c->pCurrent += n + 1;
return SQLITE_OK;
}
c->pCurrent += n + 1;
/* TODO(shess) could strspn() to skip delimiters en masse. Needs
** to happen in two places, though, which is annoying.
*/
}
return SQLITE_DONE;
}
static sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
void get_simple_tokenizer_module(
sqlite3_tokenizer_module **ppModule
){
*ppModule = &simpleTokenizerModule;
}
+89
View File
@@ -0,0 +1,89 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _TOKENIZER_H_
#define _TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface.
*/
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
struct sqlite3_tokenizer_module {
int iVersion; /* currently 0 */
/*
** Create and destroy a tokenizer. argc/argv are passed down from
** the fulltext virtual table creation to allow customization.
*/
int (*xCreate)(int argc, const char **argv,
sqlite3_tokenizer **ppTokenizer);
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Tokenize a particular input. Call xOpen() to prepare to
** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
** xClose() to free any internal state. The pInput passed to
** xOpen() must exist until the cursor is closed. The ppToken
** result from xNext() is only valid until the next call to xNext()
** or until xClose() is called.
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xOpen)(sqlite3_tokenizer *pTokenizer,
const char *pInput, int nBytes,
sqlite3_tokenizer_cursor **ppCursor);
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
const char **ppToken, int *pnBytes,
int *piStartOffset, int *piEndOffset, int *piPosition);
};
struct sqlite3_tokenizer {
sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
/*
** Get the module for a tokenizer which generates tokens based on a
** set of non-token characters. The default is to break tokens at any
** non-alnum character, though the set of delimiters can also be
** specified by the first argv argument to xCreate().
*/
/* TODO(shess) This doesn't belong here. Need some sort of
** registration process.
*/
void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
#endif /* _TOKENIZER_H_ */
+133
View File
@@ -0,0 +1,133 @@
1. FTS2 Tokenizers
When creating a new full-text table, FTS2 allows the user to select
the text tokenizer implementation to be used when indexing text
by specifying a "tokenizer" clause as part of the CREATE VIRTUAL TABLE
statement:
CREATE VIRTUAL TABLE <table-name> USING fts2(
<columns ...> [, tokenizer <tokenizer-name> [<tokenizer-args>]]
);
The built-in tokenizers (valid values to pass as <tokenizer name>) are
"simple" and "porter".
<tokenizer-args> should consist of zero or more white-space separated
arguments to pass to the selected tokenizer implementation. The
interpretation of the arguments, if any, depends on the individual
tokenizer.
2. Custom Tokenizers
FTS2 allows users to provide custom tokenizer implementations. The
interface used to create a new tokenizer is defined and described in
the fts2_tokenizer.h source file.
Registering a new FTS2 tokenizer is similar to registering a new
virtual table module with SQLite. The user passes a pointer to a
structure containing pointers to various callback functions that
make up the implementation of the new tokenizer type. For tokenizers,
the structure (defined in fts2_tokenizer.h) is called
"sqlite3_tokenizer_module".
FTS2 does not expose a C-function that users call to register new
tokenizer types with a database handle. Instead, the pointer must
be encoded as an SQL blob value and passed to FTS2 through the SQL
engine by evaluating a special scalar function, "fts2_tokenizer()".
The fts2_tokenizer() function may be called with one or two arguments,
as follows:
SELECT fts2_tokenizer(<tokenizer-name>);
SELECT fts2_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
Where <tokenizer-name> is a string identifying the tokenizer and
<sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
structure encoded as an SQL blob. If the second argument is present,
it is registered as tokenizer <tokenizer-name> and a copy of it
returned. If only one argument is passed, a pointer to the tokenizer
implementation currently registered as <tokenizer-name> is returned,
encoded as a blob. Or, if no such tokenizer exists, an SQL exception
(error) is raised.
SECURITY: If the fts2 extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they should be
prevented from invoking the fts2_tokenizer() function, possibly using the
authorisation callback.
See "Sample code" below for an example of calling the fts2_tokenizer()
function from C code.
3. ICU Library Tokenizers
If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor
symbol defined, then there exists a built-in tokenizer named "icu"
implemented using the ICU library. The first argument passed to the
xCreate() method (see fts2_tokenizer.h) of this tokenizer may be
an ICU locale identifier. For example "tr_TR" for Turkish as used
in Turkey, or "en_AU" for English as used in Australia. For example:
"CREATE VIRTUAL TABLE thai_text USING fts2(text, tokenizer icu th_TH)"
The ICU tokenizer implementation is very simple. It splits the input
text according to the ICU rules for finding word boundaries and discards
any tokens that consist entirely of white-space. This may be suitable
for some applications in some locales, but not all. If more complex
processing is required, for example to implement stemming or
discard punctuation, this can be done by creating a tokenizer
implementation that uses the ICU tokenizer as part of its implementation.
When using the ICU tokenizer this way, it is safe to overwrite the
contents of the strings returned by the xNext() method (see
fts2_tokenizer.h).
4. Sample code.
The following two code samples illustrate the way C code should invoke
the fts2_tokenizer() scalar function:
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
int queryTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
+4
View File
@@ -0,0 +1,4 @@
This folder contains source code to the second full-text search
extension for SQLite. While the API is the same, this version uses a
substantially different storage schema from fts1, so tables will need
to be rebuilt.
+6860
View File
File diff suppressed because it is too large Load Diff
+26
View File
@@ -0,0 +1,26 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This header file is used by programs that want to link against the
** FTS2 library. All it does is declare the sqlite3Fts2Init() interface.
*/
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
int sqlite3Fts2Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
+376
View File
@@ -0,0 +1,376 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables used in SQLite.
** We've modified it slightly to serve as a standalone hash table
** implementation for the full-text indexing module.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_hash.h"
/*
** Malloc and Free functions
*/
static void *fts2HashMalloc(int n){
void *p = sqlite3_malloc(n);
if( p ){
memset(p, 0, n);
}
return p;
}
static void fts2HashFree(void *p){
sqlite3_free(p);
}
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
** keyClass is one of the constants
** FTS2_HASH_BINARY or FTS2_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
void sqlite3Fts2HashInit(fts2Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS2_HASH_STRING && keyClass<=FTS2_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
pNew->count = 0;
pNew->htsize = 0;
pNew->ht = 0;
}
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
void sqlite3Fts2HashClear(fts2Hash *pH){
fts2HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
fts2HashFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
fts2HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
fts2HashFree(elem->pKey);
}
fts2HashFree(elem);
elem = next_elem;
}
pH->count = 0;
}
/*
** Hash and comparison functions when the mode is FTS2_HASH_STRING
*/
static int strHash(const void *pKey, int nKey){
const char *z = (const char *)pKey;
int h = 0;
if( nKey<=0 ) nKey = (int) strlen(z);
while( nKey > 0 ){
h = (h<<3) ^ h ^ *z++;
nKey--;
}
return h & 0x7fffffff;
}
static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
/*
** Hash and comparison functions when the mode is FTS2_HASH_BINARY
*/
static int binHash(const void *pKey, int nKey){
int h = 0;
const char *z = (const char *)pKey;
while( nKey-- > 0 ){
h = (h<<3) ^ h ^ *(z++);
}
return h & 0x7fffffff;
}
static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
if( n1!=n2 ) return 1;
return memcmp(pKey1,pKey2,n1);
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "hashFunction". The function takes a
** single parameter "keyClass". The return value of hashFunction()
** is a pointer to another function. Specifically, the return value
** of hashFunction() is a pointer to a function that takes two parameters
** with types "const void*" and "int" and returns an "int".
*/
static int (*hashFunction(int keyClass))(const void*,int){
if( keyClass==FTS2_HASH_STRING ){
return &strHash;
}else{
assert( keyClass==FTS2_HASH_BINARY );
return &binHash;
}
}
/*
** Return a pointer to the appropriate hash function given the key class.
**
** For help in interpreted the obscure C code in the function definition,
** see the header comment on the previous function.
*/
static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
if( keyClass==FTS2_HASH_STRING ){
return &strCompare;
}else{
assert( keyClass==FTS2_HASH_BINARY );
return &binCompare;
}
}
/* Link an element into the hash table
*/
static void insertElement(
fts2Hash *pH, /* The complete hash table */
struct _fts2ht *pEntry, /* The entry into which pNew is inserted */
fts2HashElem *pNew /* The element to be inserted */
){
fts2HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
else { pH->first = pNew; }
pHead->prev = pNew;
}else{
pNew->next = pH->first;
if( pH->first ){ pH->first->prev = pNew; }
pNew->prev = 0;
pH->first = pNew;
}
pEntry->count++;
pEntry->chain = pNew;
}
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
static void rehash(fts2Hash *pH, int new_size){
struct _fts2ht *new_ht; /* The new hash table */
fts2HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts2ht *)fts2HashMalloc( new_size*sizeof(struct _fts2ht) );
if( new_ht==0 ) return;
fts2HashFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
xHash = hashFunction(pH->keyClass);
for(elem=pH->first, pH->first=0; elem; elem = next_elem){
int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
next_elem = elem->next;
insertElement(pH, &new_ht[h], elem);
}
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
static fts2HashElem *findElementGivenHash(
const fts2Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
fts2HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts2ht *pEntry = &pH->ht[h];
elem = pEntry->chain;
count = pEntry->count;
xCompare = compareFunction(pH->keyClass);
while( count-- && elem ){
if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
}
}
return 0;
}
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
fts2Hash *pH, /* The pH containing "elem" */
fts2HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts2ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
if( elem->next ){
elem->next->prev = elem->prev;
}
pEntry = &pH->ht[h];
if( pEntry->chain==elem ){
pEntry->chain = elem->next;
}
pEntry->count--;
if( pEntry->count<=0 ){
pEntry->chain = 0;
}
if( pH->copyKey && elem->pKey ){
fts2HashFree(elem->pKey);
}
fts2HashFree( elem );
pH->count--;
if( pH->count<=0 ){
assert( pH->first==0 );
assert( pH->count==0 );
fts2HashClear(pH);
}
}
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
fts2HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
return elem ? elem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created. A copy of the key is made if the copyKey
** flag is set. NULL is returned.
**
** If another element already exists with the same key, then the
** new data replaces the old data and the old data is returned.
** The key is not copied in this instance. If a malloc fails, then
** the new data is returned and the hash table is unchanged.
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts2HashInsert(
fts2Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
fts2HashElem *elem; /* Used to loop thru the element list */
fts2HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = hashFunction(pH->keyClass);
assert( xHash!=0 );
hraw = (*xHash)(pKey, nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
elem = findElementGivenHash(pH,pKey,nKey,h);
if( elem ){
void *old_data = elem->data;
if( data==0 ){
removeElementGivenHash(pH,elem,h);
}else{
elem->data = data;
}
return old_data;
}
if( data==0 ) return 0;
new_elem = (fts2HashElem*)fts2HashMalloc( sizeof(fts2HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = fts2HashMalloc( nKey );
if( new_elem->pKey==0 ){
fts2HashFree(new_elem);
return data;
}
memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
new_elem->pKey = (void*)pKey;
}
new_elem->nKey = nKey;
pH->count++;
if( pH->htsize==0 ){
rehash(pH,8);
if( pH->htsize==0 ){
pH->count = 0;
fts2HashFree(new_elem);
return data;
}
}
if( pH->count > pH->htsize ){
rehash(pH,pH->htsize*2);
}
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
insertElement(pH, &pH->ht[h], new_elem);
new_elem->data = data;
return 0;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
+110
View File
@@ -0,0 +1,110 @@
/*
** 2001 September 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implementation
** used in SQLite. We've modified it slightly to serve as a standalone
** hash table implementation for the full-text indexing module.
**
*/
#ifndef _FTS2_HASH_H_
#define _FTS2_HASH_H_
/* Forward declarations of structures. */
typedef struct fts2Hash fts2Hash;
typedef struct fts2HashElem fts2HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** code should not attempt to access or modify the fields of this structure
** directly. Change this structure only by using the routines below.
** However, many of the "procedures" and "functions" for modifying and
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
struct fts2Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
fts2HashElem *first; /* The first element of the array */
int htsize; /* Number of buckets in the hash table */
struct _fts2ht { /* the hash table */
int count; /* Number of entries with this hash */
fts2HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
struct fts2HashElem {
fts2HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
** FTS2_HASH_STRING pKey points to a string that is nKey bytes long
** (including the null-terminator, if any). Case
** is respected in comparisons.
**
** FTS2_HASH_BINARY pKey points to binary data nKey bytes long.
** memcmp() is used to compare keys.
**
** A copy of the key is made if the copyKey parameter to fts2HashInit is 1.
*/
#define FTS2_HASH_STRING 1
#define FTS2_HASH_BINARY 2
/*
** Access routines. To delete, insert a NULL pointer.
*/
void sqlite3Fts2HashInit(fts2Hash*, int keytype, int copyKey);
void *sqlite3Fts2HashInsert(fts2Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts2HashFind(const fts2Hash*, const void *pKey, int nKey);
void sqlite3Fts2HashClear(fts2Hash*);
/*
** Shorthand for the functions above
*/
#define fts2HashInit sqlite3Fts2HashInit
#define fts2HashInsert sqlite3Fts2HashInsert
#define fts2HashFind sqlite3Fts2HashFind
#define fts2HashClear sqlite3Fts2HashClear
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
** fts2Hash h;
** fts2HashElem *p;
** ...
** for(p=fts2HashFirst(&h); p; p=fts2HashNext(p)){
** SomeStructure *pData = fts2HashData(p);
** // do something with pData
** }
*/
#define fts2HashFirst(H) ((H)->first)
#define fts2HashNext(E) ((E)->next)
#define fts2HashData(E) ((E)->data)
#define fts2HashKey(E) ((E)->pKey)
#define fts2HashKeysize(E) ((E)->nKey)
/*
** Number of entries in a hash table
*/
#define fts2HashCount(H) ((H)->count)
#endif /* _FTS2_HASH_H_ */
+260
View File
@@ -0,0 +1,260 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements a tokenizer for fts2 based on the ICU library.
**
** $Id: fts2_icu.c,v 1.3 2008/12/18 05:30:26 danielk1977 Exp $
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#ifdef SQLITE_ENABLE_ICU
#include <assert.h>
#include <string.h>
#include "fts2_tokenizer.h"
#include <unicode/ubrk.h>
#include <unicode/ucol.h>
#include <unicode/ustring.h>
#include <unicode/utf16.h>
typedef struct IcuTokenizer IcuTokenizer;
typedef struct IcuCursor IcuCursor;
struct IcuTokenizer {
sqlite3_tokenizer base;
char *zLocale;
};
struct IcuCursor {
sqlite3_tokenizer_cursor base;
UBreakIterator *pIter; /* ICU break-iterator object */
int nChar; /* Number of UChar elements in pInput */
UChar *aChar; /* Copy of input using utf-16 encoding */
int *aOffset; /* Offsets of each character in utf-8 input */
int nBuffer;
char *zBuffer;
int iToken;
};
/*
** Create a new tokenizer instance.
*/
static int icuCreate(
int argc, /* Number of entries in argv[] */
const char * const *argv, /* Tokenizer creation arguments */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
){
IcuTokenizer *p;
int n = 0;
if( argc>0 ){
n = strlen(argv[0])+1;
}
p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
if( !p ){
return SQLITE_NOMEM;
}
memset(p, 0, sizeof(IcuTokenizer));
if( n ){
p->zLocale = (char *)&p[1];
memcpy(p->zLocale, argv[0], n);
}
*ppTokenizer = (sqlite3_tokenizer *)p;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int icuDestroy(sqlite3_tokenizer *pTokenizer){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
sqlite3_free(p);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int icuOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, /* Input string */
int nInput, /* Length of zInput in bytes */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
IcuCursor *pCsr;
const int32_t opt = U_FOLD_CASE_DEFAULT;
UErrorCode status = U_ZERO_ERROR;
int nChar;
UChar32 c;
int iInput = 0;
int iOut = 0;
*ppCursor = 0;
if( nInput<0 ){
nInput = strlen(zInput);
}
nChar = nInput+1;
pCsr = (IcuCursor *)sqlite3_malloc(
sizeof(IcuCursor) + /* IcuCursor */
((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */
(nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
);
if( !pCsr ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(IcuCursor));
pCsr->aChar = (UChar *)&pCsr[1];
pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3];
pCsr->aOffset[iOut] = iInput;
U8_NEXT(zInput, iInput, nInput, c);
while( c>0 ){
int isError = 0;
c = u_foldCase(c, opt);
U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
if( isError ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->aOffset[iOut] = iInput;
if( iInput<nInput ){
U8_NEXT(zInput, iInput, nInput, c);
}else{
c = 0;
}
}
pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
if( !U_SUCCESS(status) ){
sqlite3_free(pCsr);
return SQLITE_ERROR;
}
pCsr->nChar = iOut;
ubrk_first(pCsr->pIter);
*ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to icuOpen().
*/
static int icuClose(sqlite3_tokenizer_cursor *pCursor){
IcuCursor *pCsr = (IcuCursor *)pCursor;
ubrk_close(pCsr->pIter);
sqlite3_free(pCsr->zBuffer);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor.
*/
static int icuNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
IcuCursor *pCsr = (IcuCursor *)pCursor;
int iStart = 0;
int iEnd = 0;
int nByte = 0;
while( iStart==iEnd ){
UChar32 c;
iStart = ubrk_current(pCsr->pIter);
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
}
do {
UErrorCode status = U_ZERO_ERROR;
if( nByte ){
char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
if( !zNew ){
return SQLITE_NOMEM;
}
pCsr->zBuffer = zNew;
pCsr->nBuffer = nByte;
}
u_strToUTF8(
pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
&pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
&status /* Output success/failure */
);
} while( nByte>pCsr->nBuffer );
*ppToken = pCsr->zBuffer;
*pnBytes = nByte;
*piStartOffset = pCsr->aOffset[iStart];
*piEndOffset = pCsr->aOffset[iEnd];
*piPosition = pCsr->iToken++;
return SQLITE_OK;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
};
/*
** Set *ppModule to point at the implementation of the ICU tokenizer.
*/
void sqlite3Fts2IcuTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &icuTokenizerModule;
}
#endif /* defined(SQLITE_ENABLE_ICU) */
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
+644
View File
@@ -0,0 +1,644 @@
/*
** 2006 September 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_tokenizer.h"
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
sqlite3_tokenizer base; /* Base class */
} porter_tokenizer;
/*
** Class derived from sqlit3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *zInput; /* input we are tokenizing */
int nInput; /* size of the input */
int iOffset; /* current position in zInput */
int iToken; /* index of next token to be returned */
char *zToken; /* storage for current token */
int nAllocated; /* space allocated to zToken buffer */
} porter_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module porterTokenizerModule;
/*
** Create a new tokenizer instance.
*/
static int porterCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
porter_tokenizer *t;
t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int porterDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is zInput[0..nInput-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int porterOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *zInput, int nInput, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
porter_tokenizer_cursor *c;
c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->zInput = zInput;
if( zInput==0 ){
c->nInput = 0;
}else if( nInput<0 ){
c->nInput = (int)strlen(zInput);
}else{
c->nInput = nInput;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->zToken = NULL; /* no space allocated, yet. */
c->nAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** porterOpen() above.
*/
static int porterClose(sqlite3_tokenizer_cursor *pCursor){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
sqlite3_free(c->zToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Vowel or consonant
*/
static const char cType[] = {
0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
1, 1, 1, 2, 1
};
/*
** isConsonant() and isVowel() determine if their first character in
** the string they point to is a consonant or a vowel, according
** to Porter ruls.
**
** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
** 'Y' is a consonant unless it follows another consonant,
** in which case it is a vowel.
**
** In these routine, the letters are in reverse order. So the 'y' rule
** is that 'y' is a consonant unless it is followed by another
** consonent.
*/
static int isVowel(const char*);
static int isConsonant(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return j;
return z[1]==0 || isVowel(z + 1);
}
static int isVowel(const char *z){
int j;
char x = *z;
if( x==0 ) return 0;
assert( x>='a' && x<='z' );
j = cType[x-'a'];
if( j<2 ) return 1-j;
return isConsonant(z + 1);
}
/*
** Let any sequence of one or more vowels be represented by V and let
** C be sequence of one or more consonants. Then every word can be
** represented as:
**
** [C] (VC){m} [V]
**
** In prose: A word is an optional consonant followed by zero or
** vowel-consonant pairs followed by an optional vowel. "m" is the
** number of vowel consonant pairs. This routine computes the value
** of m for the first i bytes of a word.
**
** Return true if the m-value for z is 1 or more. In other words,
** return true if z contains at least one vowel that is followed
** by a consonant.
**
** In this routine z[] is in reverse order. So we are really looking
** for an instance of of a consonant followed by a vowel.
*/
static int m_gt_0(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/* Like mgt0 above except we are looking for a value of m which is
** exactly 1
*/
static int m_eq_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 1;
while( isConsonant(z) ){ z++; }
return *z==0;
}
/* Like mgt0 above except we are looking for a value of m>1 instead
** or m>0
*/
static int m_gt_1(const char *z){
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
if( *z==0 ) return 0;
while( isVowel(z) ){ z++; }
if( *z==0 ) return 0;
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if there is a vowel anywhere within z[0..n-1]
*/
static int hasVowel(const char *z){
while( isConsonant(z) ){ z++; }
return *z!=0;
}
/*
** Return TRUE if the word ends in a double consonant.
**
** The text is reversed here. So we are really looking at
** the first two characters of z[].
*/
static int doubleConsonant(const char *z){
return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
}
/*
** Return TRUE if the word ends with three letters which
** are consonant-vowel-consonent and where the final consonant
** is not 'w', 'x', or 'y'.
**
** The word is reversed here. So we are really checking the
** first three letters and the first one cannot be in [wxy].
*/
static int star_oh(const char *z){
return
z[0]!=0 && isConsonant(z) &&
z[0]!='w' && z[0]!='x' && z[0]!='y' &&
z[1]!=0 && isVowel(z+1) &&
z[2]!=0 && isConsonant(z+2);
}
/*
** If the word ends with zFrom and xCond() is true for the stem
** of the word that preceeds the zFrom ending, then change the
** ending to zTo.
**
** The input word *pz and zFrom are both in reverse order. zTo
** is in normal order.
**
** Return TRUE if zFrom matches. Return FALSE if zFrom does not
** match. Not that TRUE is returned even if xCond() fails and
** no substitution occurs.
*/
static int stem(
char **pz, /* The word being stemmed (Reversed) */
const char *zFrom, /* If the ending matches this... (Reversed) */
const char *zTo, /* ... change the ending to this (not reversed) */
int (*xCond)(const char*) /* Condition that must be true */
){
char *z = *pz;
while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
if( *zFrom!=0 ) return 0;
if( xCond && !xCond(z) ) return 1;
while( *zTo ){
*(--z) = *(zTo++);
}
*pz = z;
return 1;
}
/*
** This is the fallback stemmer used when the porter stemmer is
** inappropriate. The input word is copied into the output with
** US-ASCII case folding. If the input word is too long (more
** than 20 bytes if it contains no digits or more than 6 bytes if
** it contains digits) then word is truncated to 20 or 6 bytes
** by taking 10 or 3 bytes from the beginning and end.
*/
static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, mx, j;
int hasDigit = 0;
for(i=0; i<nIn; i++){
int c = zIn[i];
if( c>='A' && c<='Z' ){
zOut[i] = c - 'A' + 'a';
}else{
if( c>='0' && c<='9' ) hasDigit = 1;
zOut[i] = c;
}
}
mx = hasDigit ? 3 : 10;
if( nIn>mx*2 ){
for(j=mx, i=nIn-mx; i<nIn; i++, j++){
zOut[j] = zOut[i];
}
i = j;
}
zOut[i] = 0;
*pnOut = i;
}
/*
** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
** zOut is at least big enough to hold nIn bytes. Write the actual
** size of the output word (exclusive of the '\0' terminator) into *pnOut.
**
** Any upper-case characters in the US-ASCII character set ([A-Z])
** are converted to lower case. Upper-case UTF characters are
** unchanged.
**
** Words that are longer than about 20 bytes are stemmed by retaining
** a few bytes from the beginning and the end of the word. If the
** word contains digits, 3 bytes are taken from the beginning and
** 3 bytes from the end. For long words without digits, 10 bytes
** are taken from each end. US-ASCII case folding still applies.
**
** If the input word contains not digits but does characters not
** in [a-zA-Z] then no stemming is attempted and this routine just
** copies the input into the input into the output with US-ASCII
** case folding.
**
** Stemming never increases the length of the word. So there is
** no chance of overflowing the zOut buffer.
*/
static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
int i, j, c;
char zReverse[28];
char *z, *z2;
if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
c = zIn[i];
if( c>='A' && c<='Z' ){
zReverse[j] = c + 'a' - 'A';
}else if( c>='a' && c<='z' ){
zReverse[j] = c;
}else{
/* The use of a character not in [a-zA-Z] means that we fallback
** to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
return;
}
}
memset(&zReverse[sizeof(zReverse)-5], 0, 5);
z = &zReverse[j+1];
/* Step 1a */
if( z[0]=='s' ){
if(
!stem(&z, "sess", "ss", 0) &&
!stem(&z, "sei", "i", 0) &&
!stem(&z, "ss", "ss", 0)
){
z++;
}
}
/* Step 1b */
z2 = z;
if( stem(&z, "dee", "ee", m_gt_0) ){
/* Do nothing. The work was all in the test */
}else if(
(stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
&& z!=z2
){
if( stem(&z, "ta", "ate", 0) ||
stem(&z, "lb", "ble", 0) ||
stem(&z, "zi", "ize", 0) ){
/* Do nothing. The work was all in the test */
}else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
z++;
}else if( m_eq_1(z) && star_oh(z) ){
*(--z) = 'e';
}
}
/* Step 1c */
if( z[0]=='y' && hasVowel(z+1) ){
z[0] = 'i';
}
/* Step 2 */
switch( z[1] ){
case 'a':
stem(&z, "lanoita", "ate", m_gt_0) ||
stem(&z, "lanoit", "tion", m_gt_0);
break;
case 'c':
stem(&z, "icne", "ence", m_gt_0) ||
stem(&z, "icna", "ance", m_gt_0);
break;
case 'e':
stem(&z, "rezi", "ize", m_gt_0);
break;
case 'g':
stem(&z, "igol", "log", m_gt_0);
break;
case 'l':
stem(&z, "ilb", "ble", m_gt_0) ||
stem(&z, "illa", "al", m_gt_0) ||
stem(&z, "iltne", "ent", m_gt_0) ||
stem(&z, "ile", "e", m_gt_0) ||
stem(&z, "ilsuo", "ous", m_gt_0);
break;
case 'o':
stem(&z, "noitazi", "ize", m_gt_0) ||
stem(&z, "noita", "ate", m_gt_0) ||
stem(&z, "rota", "ate", m_gt_0);
break;
case 's':
stem(&z, "msila", "al", m_gt_0) ||
stem(&z, "ssenevi", "ive", m_gt_0) ||
stem(&z, "ssenluf", "ful", m_gt_0) ||
stem(&z, "ssensuo", "ous", m_gt_0);
break;
case 't':
stem(&z, "itila", "al", m_gt_0) ||
stem(&z, "itivi", "ive", m_gt_0) ||
stem(&z, "itilib", "ble", m_gt_0);
break;
}
/* Step 3 */
switch( z[0] ){
case 'e':
stem(&z, "etaci", "ic", m_gt_0) ||
stem(&z, "evita", "", m_gt_0) ||
stem(&z, "ezila", "al", m_gt_0);
break;
case 'i':
stem(&z, "itici", "ic", m_gt_0);
break;
case 'l':
stem(&z, "laci", "ic", m_gt_0) ||
stem(&z, "luf", "", m_gt_0);
break;
case 's':
stem(&z, "ssen", "", m_gt_0);
break;
}
/* Step 4 */
switch( z[1] ){
case 'a':
if( z[0]=='l' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'c':
if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'e':
if( z[0]=='r' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'i':
if( z[0]=='c' && m_gt_1(z+2) ){
z += 2;
}
break;
case 'l':
if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
z += 4;
}
break;
case 'n':
if( z[0]=='t' ){
if( z[2]=='a' ){
if( m_gt_1(z+3) ){
z += 3;
}
}else if( z[2]=='e' ){
stem(&z, "tneme", "", m_gt_1) ||
stem(&z, "tnem", "", m_gt_1) ||
stem(&z, "tne", "", m_gt_1);
}
}
break;
case 'o':
if( z[0]=='u' ){
if( m_gt_1(z+2) ){
z += 2;
}
}else if( z[3]=='s' || z[3]=='t' ){
stem(&z, "noi", "", m_gt_1);
}
break;
case 's':
if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
case 't':
stem(&z, "eta", "", m_gt_1) ||
stem(&z, "iti", "", m_gt_1);
break;
case 'u':
if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
z += 3;
}
break;
case 'v':
case 'z':
if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
z += 3;
}
break;
}
/* Step 5a */
if( z[0]=='e' ){
if( m_gt_1(z+1) ){
z++;
}else if( m_eq_1(z+1) && !star_oh(z+1) ){
z++;
}
}
/* Step 5b */
if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
z++;
}
/* z[] is now the stemmed word in reverse order. Flip it back
** around into forward order and return.
*/
*pnOut = i = strlen(z);
zOut[i] = 0;
while( *z ){
zOut[--i] = *(z++);
}
}
/*
** Characters that can be part of a token. We assume any character
** whose value is greater than 0x80 (any UTF character) can be
** part of a token. In other words, delimiters all must have
** values of 0x7f or lower.
*/
static const char porterIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to porterOpen().
*/
static int porterNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
const char **pzToken, /* OUT: *pzToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
const char *z = c->zInput;
while( c->iOffset<c->nInput ){
int iStartOffset, ch;
/* Scan past delimiter characters */
while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int n = c->iOffset-iStartOffset;
if( n>c->nAllocated ){
c->nAllocated = n+20;
c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
if( c->zToken==NULL ) return SQLITE_NOMEM;
}
porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
*pzToken = c->zToken;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the porter-stemmer tokenizer
*/
static const sqlite3_tokenizer_module porterTokenizerModule = {
0,
porterCreate,
porterDestroy,
porterOpen,
porterClose,
porterNext,
};
/*
** Allocate a new porter tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts2PorterTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &porterTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
+375
View File
@@ -0,0 +1,375 @@
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is part of an SQLite module implementing full-text search.
** This particular file implements the generic tokenizer interface.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_hash.h"
#include "fts2_tokenizer.h"
#include <assert.h>
/*
** Implementation of the SQL scalar function for accessing the underlying
** hash table. This function may be called as follows:
**
** SELECT <function-name>(<key-name>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
**
** If the <pointer> argument is specified, it must be a blob value
** containing a pointer to be stored as the hash data corresponding
** to the string <key-name>. If <pointer> is not specified, then
** the string <key-name> must already exist in the has table. Otherwise,
** an error is returned.
**
** Whether or not the <pointer> argument is specified, the value returned
** is a blob containing the pointer stored as the hash data corresponding
** to string <key-name> (after the hash-table is updated, if applicable).
*/
static void scalarFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
fts2Hash *pHash;
void *pPtr = 0;
const unsigned char *zName;
int nName;
assert( argc==1 || argc==2 );
pHash = (fts2Hash *)sqlite3_user_data(context);
zName = sqlite3_value_text(argv[0]);
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( n!=sizeof(pPtr) ){
sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
pPtr = *(void **)sqlite3_value_blob(argv[1]);
pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
if( pOld==pPtr ){
sqlite3_result_error(context, "out of memory", -1);
return;
}
}else{
pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
if( !pPtr ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
}
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
#ifdef SQLITE_TEST
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <string.h>
/*
** Implementation of a special SQL scalar function for testing tokenizers
** designed to be used in concert with the Tcl testing framework. This
** function must be called with two arguments:
**
** SELECT <function-name>(<key-name>, <input-string>);
** SELECT <function-name>(<key-name>, <pointer>);
**
** where <function-name> is the name passed as the second argument
** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
**
** The return value is a string that may be interpreted as a Tcl
** list. For each token in the <input-string>, three elements are
** added to the returned list. The first is the token position, the
** second is the token text (folded, stemmed, etc.) and the third is the
** substring of <input-string> associated with the token. For example,
** using the built-in "simple" tokenizer:
**
** SELECT fts_tokenizer_test('simple', 'I don't see how');
**
** will return the string:
**
** "{0 i I 1 dont don't 2 see see 3 how how}"
**
*/
static void testFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
fts2Hash *pHash;
sqlite3_tokenizer_module *p;
sqlite3_tokenizer *pTokenizer = 0;
sqlite3_tokenizer_cursor *pCsr = 0;
const char *zErr = 0;
const char *zName;
int nName;
const char *zInput;
int nInput;
const char *zArg = 0;
const char *zToken;
int nToken;
int iStart;
int iEnd;
int iPos;
Tcl_Obj *pRet;
assert( argc==2 || argc==3 );
nName = sqlite3_value_bytes(argv[0]);
zName = (const char *)sqlite3_value_text(argv[0]);
nInput = sqlite3_value_bytes(argv[argc-1]);
zInput = (const char *)sqlite3_value_text(argv[argc-1]);
if( argc==3 ){
zArg = (const char *)sqlite3_value_text(argv[1]);
}
pHash = (fts2Hash *)sqlite3_user_data(context);
p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
if( !p ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
sqlite3_free(zErr);
return;
}
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
zErr = "error in xCreate()";
goto finish;
}
pTokenizer->pModule = p;
if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
zErr = "error in xOpen()";
goto finish;
}
pCsr->pTokenizer = pTokenizer;
while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
zToken = &zInput[iStart];
nToken = iEnd-iStart;
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
}
if( SQLITE_OK!=p->xClose(pCsr) ){
zErr = "error in xClose()";
goto finish;
}
if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
zErr = "error in xDestroy()";
goto finish;
}
finish:
if( zErr ){
sqlite3_result_error(context, zErr, -1);
}else{
sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
}
Tcl_DecrRefCount(pRet);
}
static
int registerTokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module *p
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
sqlite3_step(pStmt);
return sqlite3_finalize(pStmt);
}
static
int queryFts2Tokenizer(
sqlite3 *db,
char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts2_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
/*
** Implementation of the scalar function fts2_tokenizer_internal_test().
** This function is used for testing only, it is not included in the
** build unless SQLITE_TEST is defined.
**
** The purpose of this is to test that the fts2_tokenizer() function
** can be used as designed by the C-code in the queryFts2Tokenizer and
** registerTokenizer() functions above. These two functions are repeated
** in the README.tokenizer file as an example, so it is important to
** test them.
**
** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
** function with no arguments. An assert() will fail if a problem is
** detected. i.e.:
**
** SELECT fts2_tokenizer_internal_test();
**
*/
static void intTestFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int rc;
const sqlite3_tokenizer_module *p1;
const sqlite3_tokenizer_module *p2;
sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
/* Test the query function */
sqlite3Fts2SimpleTokenizerModule(&p1);
rc = queryFts2Tokenizer(db, "simple", &p2);
assert( rc==SQLITE_OK );
assert( p1==p2 );
rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_ERROR );
assert( p2==0 );
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
#endif
/*
** Set up SQL objects in database db used to access the contents of
** the hash table pointed to by argument pHash. The hash table must
** been initialized to use string keys, and to take a private copy
** of the key when a value is inserted. i.e. by a call similar to:
**
** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
**
** This function adds a scalar function (see header comment above
** scalarFunc() in this file for details) and, if ENABLE_TABLE is
** defined at compilation time, a temporary virtual table (see header
** comment above struct HashTableVtab) to the database schema. Both
** provide read/write access to the contents of *pHash.
**
** The third argument to this function, zName, is used as the name
** of both the scalar and, if created, the virtual table.
*/
int sqlite3Fts2InitHashTable(
sqlite3 *db,
fts2Hash *pHash,
const char *zName
){
int rc = SQLITE_OK;
void *p = (void *)pHash;
const int any = SQLITE_ANY;
char *zTest = 0;
char *zTest2 = 0;
#ifdef SQLITE_TEST
void *pdb = (void *)db;
zTest = sqlite3_mprintf("%s_test", zName);
zTest2 = sqlite3_mprintf("%s_internal_test", zName);
if( !zTest || !zTest2 ){
rc = SQLITE_NOMEM;
}
#endif
if( rc!=SQLITE_OK
|| (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
#ifdef SQLITE_TEST
|| (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
|| (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
#endif
);
sqlite3_free(zTest);
sqlite3_free(zTest2);
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
+145
View File
@@ -0,0 +1,145 @@
/*
** 2006 July 10
**
** The author disclaims copyright to this source code.
**
*************************************************************************
** Defines the interface to tokenizers used by fulltext-search. There
** are three basic components:
**
** sqlite3_tokenizer_module is a singleton defining the tokenizer
** interface functions. This is essentially the class structure for
** tokenizers.
**
** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
** including customization information defined at creation time.
**
** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
** tokens from a particular input.
*/
#ifndef _FTS2_TOKENIZER_H_
#define _FTS2_TOKENIZER_H_
/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
** If tokenizers are to be allowed to call sqlite3_*() functions, then
** we will need a way to register the API consistently.
*/
#include "sqlite3.h"
/*
** Structures used by the tokenizer interface. When a new tokenizer
** implementation is registered, the caller provides a pointer to
** an sqlite3_tokenizer_module containing pointers to the callback
** functions that make up an implementation.
**
** When an fts2 table is created, it passes any arguments passed to
** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
** implementation. The xCreate() function in turn returns an
** sqlite3_tokenizer structure representing the specific tokenizer to
** be used for the fts2 table (customized by the tokenizer clause arguments).
**
** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
** method is called. It returns an sqlite3_tokenizer_cursor object
** that may be used to tokenize a specific input buffer based on
** the tokenization rules supplied by a specific sqlite3_tokenizer
** object.
*/
typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
typedef struct sqlite3_tokenizer sqlite3_tokenizer;
typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
struct sqlite3_tokenizer_module {
/*
** Structure version. Should always be set to 0.
*/
int iVersion;
/*
** Create a new tokenizer. The values in the argv[] array are the
** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
** TABLE statement that created the fts2 table. For example, if
** the following SQL is executed:
**
** CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2)
**
** then argc is set to 2, and the argv[] array contains pointers
** to the strings "arg1" and "arg2".
**
** This method should return either SQLITE_OK (0), or an SQLite error
** code. If SQLITE_OK is returned, then *ppTokenizer should be set
** to point at the newly created tokenizer structure. The generic
** sqlite3_tokenizer.pModule variable should not be initialized by
** this callback. The caller will do so.
*/
int (*xCreate)(
int argc, /* Size of argv array */
const char *const*argv, /* Tokenizer argument strings */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
);
/*
** Destroy an existing tokenizer. The fts2 module calls this method
** exactly once for each successful call to xCreate().
*/
int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
/*
** Create a tokenizer cursor to tokenize an input buffer. The caller
** is responsible for ensuring that the input buffer remains valid
** until the cursor is closed (using the xClose() method).
*/
int (*xOpen)(
sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
const char *pInput, int nBytes, /* Input buffer */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
);
/*
** Destroy an existing tokenizer cursor. The fts2 module calls this
** method exactly once for each successful call to xOpen().
*/
int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
/*
** Retrieve the next token from the tokenizer cursor pCursor. This
** method should either return SQLITE_OK and set the values of the
** "OUT" variables identified below, or SQLITE_DONE to indicate that
** the end of the buffer has been reached, or an SQLite error code.
**
** *ppToken should be set to point at a buffer containing the
** normalized version of the token (i.e. after any case-folding and/or
** stemming has been performed). *pnBytes should be set to the length
** of this buffer in bytes. The input text that generated the token is
** identified by the byte offsets returned in *piStartOffset and
** *piEndOffset.
**
** The buffer *ppToken is set to point at is managed by the tokenizer
** implementation. It is only required to be valid until the next call
** to xNext() or xClose().
*/
/* TODO(shess) current implementation requires pInput to be
** nul-terminated. This should either be fixed, or pInput/nBytes
** should be converted to zInput.
*/
int (*xNext)(
sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
int *piStartOffset, /* OUT: Byte offset of token in input buffer */
int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
int *piPosition /* OUT: Number of tokens returned before this one */
);
};
struct sqlite3_tokenizer {
const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
/* Tokenizer implementations will typically add additional fields */
};
struct sqlite3_tokenizer_cursor {
sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
/* Tokenizer implementations will typically add additional fields */
};
#endif /* _FTS2_TOKENIZER_H_ */
+233
View File
@@ -0,0 +1,233 @@
/*
** 2006 Oct 10
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Implementation of the "simple" full-text-search tokenizer.
*/
/*
** The code in this file is only compiled if:
**
** * The FTS2 module is being built as an extension
** (in which case SQLITE_CORE is not defined), or
**
** * The FTS2 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "sqlite3.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#include "fts2_tokenizer.h"
typedef struct simple_tokenizer {
sqlite3_tokenizer base;
char delim[128]; /* flag ASCII delimiters */
} simple_tokenizer;
typedef struct simple_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
const char *pInput; /* input we are tokenizing */
int nBytes; /* size of the input */
int iOffset; /* current position in pInput */
int iToken; /* index of next token to be returned */
char *pToken; /* storage for current token */
int nTokenAllocated; /* space allocated to zToken buffer */
} simple_tokenizer_cursor;
/* Forward declaration */
static const sqlite3_tokenizer_module simpleTokenizerModule;
static int simpleDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
int argc, const char * const *argv,
sqlite3_tokenizer **ppTokenizer
){
simple_tokenizer *t;
t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
if( t==NULL ) return SQLITE_NOMEM;
memset(t, 0, sizeof(*t));
/* TODO(shess) Delimiters need to remain the same from run to run,
** else we need to reindex. One solution would be a meta-table to
** track such information in the database, then we'd only want this
** information on the initial create.
*/
if( argc>1 ){
int i, n = strlen(argv[1]);
for(i=0; i<n; i++){
unsigned char ch = argv[1][i];
/* We explicitly don't support UTF-8 delimiters for now. */
if( ch>=0x80 ){
sqlite3_free(t);
return SQLITE_ERROR;
}
t->delim[ch] = 1;
}
} else {
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
(i>='a' && i<='z'));
}
}
*ppTokenizer = &t->base;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
sqlite3_free(pTokenizer);
return SQLITE_OK;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int simpleOpen(
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
const char *pInput, int nBytes, /* String to be tokenized */
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
simple_tokenizer_cursor *c;
c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
if( c==NULL ) return SQLITE_NOMEM;
c->pInput = pInput;
if( pInput==0 ){
c->nBytes = 0;
}else if( nBytes<0 ){
c->nBytes = (int)strlen(pInput);
}else{
c->nBytes = nBytes;
}
c->iOffset = 0; /* start tokenizing at the beginning */
c->iToken = 0;
c->pToken = NULL; /* no space allocated, yet. */
c->nTokenAllocated = 0;
*ppCursor = &c->base;
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
sqlite3_free(c->pToken);
sqlite3_free(c);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int simpleNext(
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
const char **ppToken, /* OUT: *ppToken is the token text */
int *pnBytes, /* OUT: Number of bytes in token */
int *piStartOffset, /* OUT: Starting offset of token */
int *piEndOffset, /* OUT: Ending offset of token */
int *piPosition /* OUT: Position integer of token */
){
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
unsigned char *p = (unsigned char *)c->pInput;
while( c->iOffset<c->nBytes ){
int iStartOffset;
/* Scan past delimiter characters */
while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
/* Count non-delimiter characters. */
iStartOffset = c->iOffset;
while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
c->iOffset++;
}
if( c->iOffset>iStartOffset ){
int i, n = c->iOffset-iStartOffset;
if( n>c->nTokenAllocated ){
c->nTokenAllocated = n+20;
c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
if( c->pToken==NULL ) return SQLITE_NOMEM;
}
for(i=0; i<n; i++){
/* TODO(shess) This needs expansion to handle UTF-8
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
*piStartOffset = iStartOffset;
*piEndOffset = c->iOffset;
*piPosition = c->iToken++;
return SQLITE_OK;
}
}
return SQLITE_DONE;
}
/*
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module simpleTokenizerModule = {
0,
simpleCreate,
simpleDestroy,
simpleOpen,
simpleClose,
simpleNext,
};
/*
** Allocate a new simple tokenizer. Return a pointer to the new
** tokenizer in *ppModule
*/
void sqlite3Fts2SimpleTokenizerModule(
sqlite3_tokenizer_module const**ppModule
){
*ppModule = &simpleTokenizerModule;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/tclsh
#
# This script builds a single C code file holding all of FTS2 code.
# The name of the output file is fts2amal.c. To build this file,
# first do:
#
# make target_source
#
# The make target above moves all of the source code files into
# a subdirectory named "tsrc". (This script expects to find the files
# there and will not work if they are not found.)
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
# tclsh mkfts2amal.tcl
#
# The amalgamated FTS2 code will be written into fts2amal.c
#
# Open the output file and write a header comment at the beginning
# of the file.
#
set out [open fts2amal.c w]
set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
puts $out [subst \
{/******************************************************************************
** This file is an amalgamation of separate C source files from the SQLite
** Full Text Search extension 2 (fts2). By combining all the individual C
** code files into this single large file, the entire code can be compiled
** as a one translation unit. This allows many compilers to do optimizations
** that would not be possible if the files were compiled separately. It also
** makes the code easier to import into other projects.
**
** This amalgamation was generated on $today.
*/}]
# These are the header files used by FTS2. The first time any of these
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line. The file only needs to be included once.
#
foreach hdr {
fts2.h
fts2_hash.h
fts2_tokenizer.h
sqlite3.h
sqlite3ext.h
} {
set available_hdr($hdr) 1
}
# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}
# Insert a comment into the code
#
proc section_comment {text} {
global out s78
set n [string length $text]
set nstar [expr {60 - $n}]
set stars [string range $s78 0 $nstar]
puts $out "/************** $text $stars/"
}
# Read the source file named $filename and write it into the
# sqlite3.c output file. If any #include statements are seen,
# process them approprately.
#
proc copy_file {filename} {
global seen_hdr available_hdr out
set tail [file tail $filename]
section_comment "Begin file $tail"
set in [open $filename r]
while {![eof $in]} {
set line [gets $in]
if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
if {[info exists available_hdr($hdr)]} {
if {$available_hdr($hdr)} {
section_comment "Include $hdr in the middle of $tail"
copy_file tsrc/$hdr
section_comment "Continuing where we left off in $tail"
}
} elseif {![info exists seen_hdr($hdr)]} {
set seen_hdr($hdr) 1
puts $out $line
}
} elseif {[regexp {^#ifdef __cplusplus} $line]} {
puts $out "#if 0"
} elseif {[regexp {^#line} $line]} {
# Skip #line directives.
} else {
puts $out $line
}
}
close $in
section_comment "End of $tail"
}
# Process the source files. Process files containing commonly
# used subroutines first in order to help the compiler find
# inlining opportunities.
#
foreach file {
fts2.c
fts2_hash.c
fts2_porter.c
fts2_tokenizer.c
fts2_tokenizer1.c
fts2_icu.c
} {
copy_file tsrc/$file
}
close $out
+71 -155
View File
@@ -308,17 +308,18 @@
SQLITE_EXTENSION_INIT1
#endif
typedef struct Fts3HashWrapper Fts3HashWrapper;
struct Fts3HashWrapper {
Fts3Hash hash; /* Hash table */
int nRef; /* Number of pointers to this object */
};
static int fts3EvalNext(Fts3Cursor *pCsr);
static int fts3EvalStart(Fts3Cursor *pCsr);
static int fts3TermSegReaderCursor(
Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
#ifndef SQLITE_AMALGAMATION
# if defined(SQLITE_DEBUG)
int sqlite3Fts3Always(int b) { assert( b ); return b; }
int sqlite3Fts3Never(int b) { assert( !b ); return b; }
# endif
#endif
/*
** This variable is set to false when running tests for which the on disk
** structures should not be corrupt. Otherwise, true. If it is false, extra
@@ -640,7 +641,6 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS);
/* Create a list of user columns for the virtual table */
zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
@@ -1179,7 +1179,7 @@ static int fts3InitVtab(
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
Fts3Hash *pHash = &((Fts3HashWrapper*)pAux)->hash;
Fts3Hash *pHash = (Fts3Hash *)pAux;
Fts3Table *p = 0; /* Pointer to allocated vtab */
int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
@@ -2889,7 +2889,7 @@ static int fts3TermSelectMerge(
**
** Similar padding is added in the fts3DoclistOrMerge() function.
*/
pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1);
pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1);
pTS->anOutput[0] = nDoclist;
if( pTS->aaOutput[0] ){
memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
@@ -3890,8 +3890,6 @@ static int fts3RenameMethod(
rc = sqlite3Fts3PendingTermsFlush(p);
}
p->bIgnoreSavepoint = 1;
if( p->zContentTbl==0 ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
@@ -3919,8 +3917,6 @@ static int fts3RenameMethod(
"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';",
p->zDb, p->zName, zName
);
p->bIgnoreSavepoint = 0;
return rc;
}
@@ -3931,28 +3927,12 @@ static int fts3RenameMethod(
*/
static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
int rc = SQLITE_OK;
Fts3Table *pTab = (Fts3Table*)pVtab;
assert( pTab->inTransaction );
assert( pTab->mxSavepoint<=iSavepoint );
TESTONLY( pTab->mxSavepoint = iSavepoint );
if( pTab->bIgnoreSavepoint==0 ){
if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
pTab->zDb, pTab->zName, pTab->zName
);
if( zSql ){
pTab->bIgnoreSavepoint = 1;
rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
pTab->bIgnoreSavepoint = 0;
sqlite3_free(zSql);
}else{
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK ){
pTab->iSavepoint = iSavepoint+1;
}
UNUSED_PARAMETER(iSavepoint);
assert( ((Fts3Table *)pVtab)->inTransaction );
assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
rc = fts3SyncMethod(pVtab);
}
return rc;
}
@@ -3963,11 +3943,12 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
** This is a no-op.
*/
static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
Fts3Table *pTab = (Fts3Table*)pVtab;
assert( pTab->inTransaction );
assert( pTab->mxSavepoint >= iSavepoint );
TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
pTab->iSavepoint = iSavepoint;
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
UNUSED_PARAMETER(iSavepoint);
UNUSED_PARAMETER(pVtab);
assert( p->inTransaction );
assert( p->mxSavepoint >= iSavepoint );
TESTONLY( p->mxSavepoint = iSavepoint-1 );
return SQLITE_OK;
}
@@ -3977,13 +3958,11 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
** Discard the contents of the pending terms table.
*/
static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
Fts3Table *pTab = (Fts3Table*)pVtab;
Fts3Table *p = (Fts3Table*)pVtab;
UNUSED_PARAMETER(iSavepoint);
assert( pTab->inTransaction );
TESTONLY( pTab->mxSavepoint = iSavepoint );
if( (iSavepoint+1)<=pTab->iSavepoint ){
sqlite3Fts3PendingTermsClear(pTab);
}
assert( p->inTransaction );
TESTONLY( p->mxSavepoint = iSavepoint );
sqlite3Fts3PendingTermsClear(p);
return SQLITE_OK;
}
@@ -4002,42 +3981,8 @@ static int fts3ShadowName(const char *zName){
return 0;
}
/*
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
static int fts3IntegrityMethod(
sqlite3_vtab *pVtab, /* The virtual table to be checked */
const char *zSchema, /* Name of schema in which pVtab lives */
const char *zTabname, /* Name of the pVTab table */
int isQuick, /* True if this is a quick_check */
char **pzErr /* Write error message here */
){
Fts3Table *p = (Fts3Table*)pVtab;
int rc = SQLITE_OK;
int bOk = 0;
UNUSED_PARAMETER(isQuick);
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
assert( rc!=SQLITE_CORRUPT_VTAB );
if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS%d table %s.%s: %s",
p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
if( *pzErr ) rc = SQLITE_OK;
}else if( rc==SQLITE_OK && bOk==0 ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
p->bFts4 ? 4 : 3, zSchema, zTabname);
if( *pzErr==0 ) rc = SQLITE_NOMEM;
}
sqlite3Fts3SegmentsClose(p);
return rc;
}
static const sqlite3_module fts3Module = {
/* iVersion */ 4,
/* iVersion */ 3,
/* xCreate */ fts3CreateMethod,
/* xConnect */ fts3ConnectMethod,
/* xBestIndex */ fts3BestIndexMethod,
@@ -4061,7 +4006,6 @@ static const sqlite3_module fts3Module = {
/* xRelease */ fts3ReleaseMethod,
/* xRollbackTo */ fts3RollbackToMethod,
/* xShadowName */ fts3ShadowName,
/* xIntegrity */ fts3IntegrityMethod,
};
/*
@@ -4070,12 +4014,9 @@ static const sqlite3_module fts3Module = {
** allocated for the tokenizer hash table.
*/
static void hashDestroy(void *p){
Fts3HashWrapper *pHash = (Fts3HashWrapper *)p;
pHash->nRef--;
if( pHash->nRef<=0 ){
sqlite3Fts3HashClear(&pHash->hash);
sqlite3_free(pHash);
}
Fts3Hash *pHash = (Fts3Hash *)p;
sqlite3Fts3HashClear(pHash);
sqlite3_free(pHash);
}
/*
@@ -4105,7 +4046,7 @@ void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
*/
int sqlite3Fts3Init(sqlite3 *db){
int rc = SQLITE_OK;
Fts3HashWrapper *pHash = 0;
Fts3Hash *pHash = 0;
const sqlite3_tokenizer_module *pSimple = 0;
const sqlite3_tokenizer_module *pPorter = 0;
#ifndef SQLITE_DISABLE_FTS3_UNICODE
@@ -4133,24 +4074,23 @@ int sqlite3Fts3Init(sqlite3 *db){
sqlite3Fts3PorterTokenizerModule(&pPorter);
/* Allocate and initialize the hash-table used to store tokenizers. */
pHash = sqlite3_malloc(sizeof(Fts3HashWrapper));
pHash = sqlite3_malloc(sizeof(Fts3Hash));
if( !pHash ){
rc = SQLITE_NOMEM;
}else{
sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1);
pHash->nRef = 0;
sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
}
/* Load the built-in tokenizers into the hash table */
if( rc==SQLITE_OK ){
if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple)
|| sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter)
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|| sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode)
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
#endif
#ifdef SQLITE_ENABLE_ICU
|| (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu))
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
#endif
){
rc = SQLITE_NOMEM;
@@ -4159,7 +4099,7 @@ int sqlite3Fts3Init(sqlite3 *db){
#ifdef SQLITE_TEST
if( rc==SQLITE_OK ){
rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash);
rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
}
#endif
@@ -4168,26 +4108,23 @@ int sqlite3Fts3Init(sqlite3 *db){
** module with sqlite.
*/
if( SQLITE_OK==rc
&& SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
){
pHash->nRef++;
rc = sqlite3_create_module_v2(
db, "fts3", &fts3Module, (void *)pHash, hashDestroy
);
if( rc==SQLITE_OK ){
pHash->nRef++;
rc = sqlite3_create_module_v2(
db, "fts4", &fts3Module, (void *)pHash, hashDestroy
db, "fts4", &fts3Module, (void *)pHash, 0
);
}
if( rc==SQLITE_OK ){
pHash->nRef++;
rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy);
rc = sqlite3Fts3InitTok(db, (void *)pHash);
}
return rc;
}
@@ -4196,7 +4133,7 @@ int sqlite3Fts3Init(sqlite3 *db){
/* An error has occurred. Delete the hash table and return the error code. */
assert( rc!=SQLITE_OK );
if( pHash ){
sqlite3Fts3HashClear(&pHash->hash);
sqlite3Fts3HashClear(pHash);
sqlite3_free(pHash);
}
return rc;
@@ -4365,7 +4302,8 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
char *aPoslist = 0; /* Position list for deferred tokens */
int nPoslist = 0; /* Number of bytes in aPoslist */
int iPrev = -1; /* Token number of previous deferred token */
char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0);
assert( pPhrase->doclist.bFreeList==0 );
for(iToken=0; iToken<pPhrase->nToken; iToken++){
Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
@@ -4379,7 +4317,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
if( pList==0 ){
sqlite3_free(aPoslist);
sqlite3_free(aFree);
pPhrase->doclist.pList = 0;
pPhrase->doclist.nList = 0;
return SQLITE_OK;
@@ -4400,7 +4337,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
nPoslist = (int)(aOut - aPoslist);
if( nPoslist==0 ){
sqlite3_free(aPoslist);
sqlite3_free(aFree);
pPhrase->doclist.pList = 0;
pPhrase->doclist.nList = 0;
return SQLITE_OK;
@@ -4433,14 +4369,13 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
nDistance = iPrev - nMaxUndeferred;
}
aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING);
aOut = (char *)sqlite3_malloc(nPoslist+8);
if( !aOut ){
sqlite3_free(aPoslist);
return SQLITE_NOMEM;
}
pPhrase->doclist.pList = aOut;
assert( p1 && p2 );
if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
pPhrase->doclist.bFreeList = 1;
pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
@@ -4453,7 +4388,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
}
}
if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree);
return SQLITE_OK;
}
#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
@@ -4546,7 +4480,7 @@ void sqlite3Fts3DoclistPrev(
assert( nDoclist>0 );
assert( *pbEof==0 );
assert_fts3_nc( p || *piDocid==0 );
assert( p || *piDocid==0 );
assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
if( p==0 ){
@@ -4802,7 +4736,7 @@ static int fts3EvalIncrPhraseNext(
if( bEof==0 ){
int nList = 0;
int nByte = a[p->nToken-1].nList;
char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING);
char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING);
if( !aDoclist ) return SQLITE_NOMEM;
memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING);
@@ -5344,8 +5278,9 @@ static void fts3EvalNextRow(
Fts3Expr *pExpr, /* Expr. to advance to next matching row */
int *pRc /* IN/OUT: Error code */
){
if( *pRc==SQLITE_OK && pExpr->bEof==0 ){
if( *pRc==SQLITE_OK ){
int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */
assert( pExpr->bEof==0 );
pExpr->bStart = 1;
switch( pExpr->eType ){
@@ -5409,8 +5344,8 @@ static void fts3EvalNextRow(
Fts3Expr *pRight = pExpr->pRight;
sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
assert_fts3_nc( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
assert_fts3_nc( pRight->bStart || pLeft->iDocid==pRight->iDocid );
assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
fts3EvalNextRow(pCsr, pLeft, pRc);
@@ -5627,10 +5562,11 @@ static int fts3EvalTestExpr(
default: {
#ifndef SQLITE_DISABLE_FTS4_DEFERRED
if( pCsr->pDeferred && (pExpr->bDeferred || (
pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList
))){
if( pCsr->pDeferred
&& (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
){
Fts3Phrase *pPhrase = pExpr->pPhrase;
assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
if( pExpr->bDeferred ){
fts3EvalInvalidatePoslist(pPhrase);
}
@@ -5821,22 +5757,6 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
}
}
/*
** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array
** has not yet been allocated, allocate and zero it. Otherwise, just zero
** it.
*/
static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){
Fts3Table *pTab = (Fts3Table*)pCtx;
UNUSED_PARAMETER(iPhrase);
if( pExpr->aMI==0 ){
pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
if( pExpr->aMI==0 ) return SQLITE_NOMEM;
}
memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
return SQLITE_OK;
}
/*
** Expression pExpr must be of type FTSQUERY_PHRASE.
**
@@ -5858,6 +5778,7 @@ static int fts3EvalGatherStats(
if( pExpr->aMI==0 ){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
Fts3Expr *pRoot; /* Root of NEAR expression */
Fts3Expr *p; /* Iterator used for several purposes */
sqlite3_int64 iPrevId = pCsr->iPrevId;
sqlite3_int64 iDocid;
@@ -5865,9 +5786,7 @@ static int fts3EvalGatherStats(
/* Find the root of the NEAR expression */
pRoot = pExpr;
while( pRoot->pParent
&& (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred)
){
while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
pRoot = pRoot->pParent;
}
iDocid = pRoot->iDocid;
@@ -5875,8 +5794,14 @@ static int fts3EvalGatherStats(
assert( pRoot->bStart );
/* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab);
if( rc!=SQLITE_OK ) return rc;
for(p=pRoot; p; p=p->pLeft){
Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
assert( pE->aMI==0 );
pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
if( !pE->aMI ) return SQLITE_NOMEM;
memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
}
fts3EvalRestart(pCsr, pRoot, &rc);
while( pCsr->isEof==0 && rc==SQLITE_OK ){
@@ -6032,7 +5957,6 @@ int sqlite3Fts3EvalPhrasePoslist(
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */
int bMatch;
/* Check if this phrase descends from an OR expression node. If not,
@@ -6047,30 +5971,22 @@ int sqlite3Fts3EvalPhrasePoslist(
if( p->bEof ) bTreeEof = 1;
}
if( bOr==0 ) return SQLITE_OK;
pRun = pNear;
while( pRun->bDeferred ){
assert( pRun->pParent );
pRun = pRun->pParent;
}
/* This is the descendent of an OR node. In this case we cannot use
** an incremental phrase. Load the entire doclist for the phrase
** into memory in this case. */
if( pPhrase->bIncr ){
int bEofSave = pRun->bEof;
fts3EvalRestart(pCsr, pRun, &rc);
while( rc==SQLITE_OK && !pRun->bEof ){
fts3EvalNextRow(pCsr, pRun, &rc);
if( bEofSave==0 && pRun->iDocid==iDocid ) break;
int bEofSave = pNear->bEof;
fts3EvalRestart(pCsr, pNear, &rc);
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
if( bEofSave==0 && pNear->iDocid==iDocid ) break;
}
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){
rc = FTS_CORRUPT_VTAB;
}
}
if( bTreeEof ){
while( rc==SQLITE_OK && !pRun->bEof ){
fts3EvalNextRow(pCsr, pRun, &rc);
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
}
}
if( rc!=SQLITE_OK ) return rc;
+12 -19
View File
@@ -151,18 +151,17 @@ extern int sqlite3_fts3_may_be_corrupt;
** Macros indicating that conditional expressions are always true or
** false.
*/
#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
#endif
#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
# define ALWAYS(X) (1)
# define NEVER(X) (0)
#elif !defined(NDEBUG)
# define ALWAYS(X) ((X)?1:(assert(0),0))
# define NEVER(X) ((X)?(assert(0),1):0)
#ifdef SQLITE_COVERAGE_TEST
# define ALWAYS(x) (1)
# define NEVER(X) (0)
#elif defined(SQLITE_DEBUG)
# define ALWAYS(x) sqlite3Fts3Always((x)!=0)
# define NEVER(x) sqlite3Fts3Never((x)!=0)
int sqlite3Fts3Always(int b);
int sqlite3Fts3Never(int b);
#else
# define ALWAYS(X) (X)
# define NEVER(X) (X)
# define ALWAYS(x) (x)
# define NEVER(x) (x)
#endif
/*
@@ -265,7 +264,6 @@ struct Fts3Table {
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
int iSavepoint;
/*
** The following array of hash tables is used to buffer pending index
@@ -559,7 +557,7 @@ struct Fts3MultiSegReader {
int nAdvance; /* How many seg-readers to advance */
Fts3SegFilter *pFilter; /* Pointer to filter object */
char *aBuffer; /* Buffer to merge doclists in */
i64 nBuffer; /* Allocated size of aBuffer[] in bytes */
int nBuffer; /* Allocated size of aBuffer[] in bytes */
int iColFilter; /* If >=0, filter for this column */
int bRestart;
@@ -622,7 +620,6 @@ void sqlite3Fts3ExprFree(Fts3Expr *);
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
void *sqlite3Fts3MallocZero(i64 nByte);
int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
sqlite3_tokenizer_cursor **
@@ -642,7 +639,7 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
/* fts3_tokenize_vtab.c */
int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*));
int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
/* fts3_unicode2.c (functions generated by parsing unicode text files) */
#ifndef SQLITE_DISABLE_FTS3_UNICODE
@@ -651,9 +648,5 @@ int sqlite3FtsUnicodeIsalnum(int);
int sqlite3FtsUnicodeIsdiacritic(int);
#endif
int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */
+2 -7
View File
@@ -297,7 +297,6 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM;
memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat);
iCol = 0;
rc = SQLITE_OK;
while( i<nDoclist ){
sqlite3_int64 v = 0;
@@ -341,10 +340,6 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
/* State 3. The integer just read is a column number. */
default: assert( eState==3 );
iCol = (int)v;
if( iCol<1 ){
rc = SQLITE_CORRUPT_VTAB;
break;
}
if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM;
pCsr->aStat[iCol+1].nDoc++;
eState = 2;
@@ -353,6 +348,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
}
pCsr->iCol = 0;
rc = SQLITE_OK;
}else{
pCsr->isEof = 1;
}
@@ -545,8 +541,7 @@ int sqlite3Fts3InitAux(sqlite3 *db){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0 /* xIntegrity */
0 /* xShadowName */
};
int rc; /* Return code */
+5 -5
View File
@@ -122,7 +122,7 @@ static int fts3isspace(char c){
** zero the memory before returning a pointer to it. If unsuccessful,
** return NULL.
*/
void *sqlite3Fts3MallocZero(sqlite3_int64 nByte){
static void *fts3MallocZero(sqlite3_int64 nByte){
void *pRet = sqlite3_malloc64(nByte);
if( pRet ) memset(pRet, 0, nByte);
return pRet;
@@ -203,7 +203,7 @@ static int getNextToken(
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte);
pRet = (Fts3Expr *)fts3MallocZero(nByte);
if( !pRet ){
rc = SQLITE_NOMEM;
}else{
@@ -458,7 +458,7 @@ static int getNextNode(
if( fts3isspace(cNext)
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
pRet = (Fts3Expr *)sqlite3Fts3MallocZero(sizeof(Fts3Expr));
pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
if( !pRet ){
return SQLITE_NOMEM;
}
@@ -637,7 +637,7 @@ static int fts3ExprParse(
&& p->eType==FTSQUERY_PHRASE && pParse->isNot
){
/* Create an implicit NOT operator. */
Fts3Expr *pNot = sqlite3Fts3MallocZero(sizeof(Fts3Expr));
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
@@ -671,7 +671,7 @@ static int fts3ExprParse(
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
pAnd = sqlite3Fts3MallocZero(sizeof(Fts3Expr));
pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
+1 -1
View File
@@ -621,7 +621,7 @@ static int porterNext(
if( n>c->nAllocated ){
char *pNew;
c->nAllocated = n+20;
pNew = sqlite3_realloc64(c->zToken, c->nAllocated);
pNew = sqlite3_realloc(c->zToken, c->nAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->zToken = pNew;
}
+34 -38
View File
@@ -41,7 +41,7 @@ typedef sqlite3_int64 i64;
/*
** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
@@ -85,7 +85,7 @@ struct SnippetFragment {
};
/*
** This type is used as an sqlite3Fts3ExprIterate() context object while
** This type is used as an fts3ExprIterate() context object while
** accumulating the data returned by the matchinfo() function.
*/
typedef struct MatchInfo MatchInfo;
@@ -138,8 +138,9 @@ static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){
+ sizeof(MatchinfoBuffer);
sqlite3_int64 nStr = strlen(zMatchinfo);
pRet = sqlite3Fts3MallocZero(nByte + nStr+1);
pRet = sqlite3_malloc64(nByte + nStr+1);
if( pRet ){
memset(pRet, 0, nByte);
pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0]
+ sizeof(u32)*((int)nElem+1);
@@ -244,7 +245,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){
}
/*
** Helper function for sqlite3Fts3ExprIterate() (see below).
** Helper function for fts3ExprIterate() (see below).
*/
static int fts3ExprIterate2(
Fts3Expr *pExpr, /* Expression to iterate phrases of */
@@ -278,7 +279,7 @@ static int fts3ExprIterate2(
** Otherwise, SQLITE_OK is returned after a callback has been made for
** all eligible phrase nodes.
*/
int sqlite3Fts3ExprIterate(
static int fts3ExprIterate(
Fts3Expr *pExpr, /* Expression to iterate phrases of */
int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
@@ -287,9 +288,10 @@ int sqlite3Fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an sqlite3Fts3ExprIterate() callback used while loading the
** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** fts3ExprLoadDoclists().
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
@@ -321,9 +323,9 @@ static int fts3ExprLoadDoclists(
int *pnToken /* OUT: Number of tokens in query */
){
int rc; /* Return Code */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
sCtx.pCsr = pCsr;
rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx);
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
if( pnToken ) *pnToken = sCtx.nToken;
return rc;
@@ -336,7 +338,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
int nPhrase = 0;
(void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
(void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
return nPhrase;
}
@@ -398,7 +400,6 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){
return 1;
}
assert( pIter->nSnippet>=0 );
pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
for(i=0; i<pIter->nPhrase; i++){
SnippetPhrase *pPhrase = &pIter->aPhrase[i];
@@ -447,7 +448,7 @@ static void fts3SnippetDetails(
}
mCover |= mPhrase;
for(j=0; j<pPhrase->nToken && j<pIter->nSnippet; j++){
for(j=0; j<pPhrase->nToken; j++){
mHighlight |= (mPos>>j);
}
@@ -465,9 +466,8 @@ static void fts3SnippetDetails(
}
/*
** This function is an sqlite3Fts3ExprIterate() callback used by
** fts3BestSnippet(). Each invocation populates an element of the
** SnippetIter.aPhrase[] array.
** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
** Each invocation populates an element of the SnippetIter.aPhrase[] array.
*/
static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
SnippetIter *p = (SnippetIter *)ctx;
@@ -544,10 +544,11 @@ static int fts3BestSnippet(
** the required space using malloc().
*/
nByte = sizeof(SnippetPhrase) * nList;
sIter.aPhrase = (SnippetPhrase *)sqlite3Fts3MallocZero(nByte);
sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte);
if( !sIter.aPhrase ){
return SQLITE_NOMEM;
}
memset(sIter.aPhrase, 0, nByte);
/* Initialize the contents of the SnippetIter object. Then iterate through
** the set of phrases in the expression to populate the aPhrase[] array.
@@ -557,9 +558,7 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
rc = sqlite3Fts3ExprIterate(
pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter
);
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
if( rc==SQLITE_OK ){
/* Set the *pmSeen output variable. */
@@ -920,10 +919,10 @@ static int fts3ExprLHitGather(
}
/*
** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo
** stats for a single query.
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
**
** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a
** fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
** of the matchinfo array that are constant for all rows returned by the
** current query.
@@ -958,7 +957,7 @@ static int fts3ExprGlobalHitsCb(
}
/*
** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the
** fts3ExprIterate() callback used to collect the "local" part of the
** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
** array that are different for each row returned by the query.
*/
@@ -1113,12 +1112,10 @@ static int fts3MatchinfoLcsCb(
** position list for the next column.
*/
static int fts3LcsIteratorAdvance(LcsIterator *pIter){
char *pRead;
char *pRead = pIter->pRead;
sqlite3_int64 iRead;
int rc = 0;
if( NEVER(pIter==0) ) return 1;
pRead = pIter->pRead;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
if( iRead==0 || iRead==1 ){
pRead = 0;
@@ -1152,9 +1149,10 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
/* Allocate and populate the array of LcsIterator objects. The array
** contains one element for each matchable phrase in the query.
**/
aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase);
aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase);
if( !aIter ) return SQLITE_NOMEM;
(void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
@@ -1331,11 +1329,11 @@ static int fts3MatchinfoValues(
rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
if( rc!=SQLITE_OK ) break;
}
rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
sqlite3Fts3EvalTestDeferred(pCsr, &rc);
if( rc!=SQLITE_OK ) break;
}
(void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
(void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
break;
}
}
@@ -1558,7 +1556,7 @@ struct TermOffsetCtx {
};
/*
** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets().
** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
*/
static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
TermOffsetCtx *p = (TermOffsetCtx *)ctx;
@@ -1614,7 +1612,7 @@ void sqlite3Fts3Offsets(
if( rc!=SQLITE_OK ) goto offsets_out;
/* Allocate the array of TermOffset iterators. */
sCtx.aTerm = (TermOffset *)sqlite3Fts3MallocZero(sizeof(TermOffset)*nToken);
sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken);
if( 0==sCtx.aTerm ){
rc = SQLITE_NOMEM;
goto offsets_out;
@@ -1635,15 +1633,13 @@ void sqlite3Fts3Offsets(
const char *zDoc;
int nDoc;
/* Initialize the contents of sCtx.aTerm[] for column iCol. This
** operation may fail if the database contains corrupt records.
/* Initialize the contents of sCtx.aTerm[] for column iCol. There is
** no way that this operation can fail, so the return code from
** fts3ExprIterate() can be discarded.
*/
sCtx.iCol = iCol;
sCtx.iTerm = 0;
rc = sqlite3Fts3ExprIterate(
pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx
);
if( rc!=SQLITE_OK ) goto offsets_out;
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
/* Retreive the text stored in column iCol. If an SQL NULL is stored
** in column iCol, jump immediately to the next iteration of the loop.
+5 -14
View File
@@ -78,8 +78,6 @@ static int fts3termConnectMethod(
iIndex = atoi(argv[4]);
argc--;
}
*ppVtab = 0;
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
@@ -97,17 +95,12 @@ static int fts3termConnectMethod(
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3termTable);
p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte);
nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
p = (Fts3termTable *)sqlite3_malloc64(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, (size_t)nByte);
p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero(
sizeof(Fts3Table) + nDb + nFts3 + 2
);
if( p->pFts3Tab==0 ){
sqlite3_free(p);
return SQLITE_NOMEM;
}
p->pFts3Tab = (Fts3Table *)&p[1];
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
@@ -137,7 +130,6 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
sqlite3_finalize(pFts3->aStmt[i]);
}
sqlite3_free(pFts3->zSegmentsTbl);
sqlite3_free(pFts3);
sqlite3_free(p);
return SQLITE_OK;
}
@@ -370,8 +362,7 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0 /* xIntegrity */
0 /* xShadowName */
};
int rc; /* Return code */
+16 -13
View File
@@ -18,7 +18,14 @@
** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#include "tclsqlite.h"
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include <string.h>
#include <assert.h>
@@ -160,8 +167,7 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
Tcl_Obj *pPhrasecount = 0;
Tcl_Obj **apExprToken;
Tcl_Size nExprToken;
Tcl_Size nn;
int nExprToken;
UNUSED_PARAMETER(clientData);
@@ -195,25 +201,23 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
}
}
rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
doc.nToken = (int)nn;
rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
if( rc!=TCL_OK ) goto near_match_out;
doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
for(ii=0; ii<doc.nToken; ii++){
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
doc.aToken[ii].n = (int)nn;
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
}
rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
if( rc!=TCL_OK ) goto near_match_out;
nPhrase = (int)(nExprToken + 1) / 2;
nPhrase = (nExprToken + 1) / 2;
aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
for(ii=0; ii<nPhrase; ii++){
Tcl_Obj *pPhrase = apExprToken[ii*2];
Tcl_Obj **apToken;
Tcl_Size nToken;
int nToken;
int jj;
rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
@@ -223,12 +227,11 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
rc = TCL_ERROR;
goto near_match_out;
}
for(jj=0; jj<(int)nToken; jj++){
for(jj=0; jj<nToken; jj++){
NearToken *pT = &aPhrase[ii].aToken[jj];
pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
pT->n = (int)nn;
pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
}
aPhrase[ii].nToken = (int)nToken;
aPhrase[ii].nToken = nToken;
}
for(ii=1; ii<nPhrase; ii++){
Tcl_Obj *pNear = apExprToken[2*ii-1];
+3 -6
View File
@@ -420,7 +420,7 @@ static int fts3tokRowidMethod(
** Register the fts3tok module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
static const sqlite3_module fts3tok_module = {
0, /* iVersion */
fts3tokConnectMethod, /* xCreate */
@@ -445,14 +445,11 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0 /* xIntegrity */
0 /* xShadowName */
};
int rc; /* Return code */
rc = sqlite3_create_module_v2(
db, "fts3tokenize", &fts3tok_module, (void*)pHash, xDestroy
);
rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
return rc;
}
+5 -1
View File
@@ -226,7 +226,11 @@ int sqlite3Fts3InitTokenizer(
#ifdef SQLITE_TEST
#include "tclsqlite.h"
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include <string.h>
/*
+1 -1
View File
@@ -185,7 +185,7 @@ static int simpleNext(
if( n>c->nTokenAllocated ){
char *pNew;
c->nTokenAllocated = n+20;
pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated);
pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->pToken = pNew;
}
+50 -87
View File
@@ -649,7 +649,7 @@ static int fts3PendingListAppendVarint(
/* Allocate or grow the PendingList as required. */
if( !p ){
p = sqlite3_malloc64(sizeof(*p) + 100);
p = sqlite3_malloc(sizeof(*p) + 100);
if( !p ){
return SQLITE_NOMEM;
}
@@ -658,14 +658,14 @@ static int fts3PendingListAppendVarint(
p->nData = 0;
}
else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
i64 nNew = p->nSpace * 2;
p = sqlite3_realloc64(p, sizeof(*p) + nNew);
int nNew = p->nSpace * 2;
p = sqlite3_realloc(p, sizeof(*p) + nNew);
if( !p ){
sqlite3_free(*pp);
*pp = 0;
return SQLITE_NOMEM;
}
p->nSpace = (int)nNew;
p->nSpace = nNew;
p->aData = (char *)&p[1];
}
@@ -1222,7 +1222,7 @@ int sqlite3Fts3ReadBlock(
int nByte = sqlite3_blob_bytes(p->pSegments);
*pnBlob = nByte;
if( paBlob ){
char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING);
char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
if( !aByte ){
rc = SQLITE_NOMEM;
}else{
@@ -1335,19 +1335,9 @@ static int fts3SegReaderNext(
char *aCopy;
PendingList *pList = (PendingList *)fts3HashData(pElem);
int nCopy = pList->nData+1;
int nTerm = fts3HashKeysize(pElem);
if( (nTerm+1)>pReader->nTermAlloc ){
sqlite3_free(pReader->zTerm);
pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2);
if( !pReader->zTerm ) return SQLITE_NOMEM;
pReader->nTermAlloc = (nTerm+1)*2;
}
memcpy(pReader->zTerm, fts3HashKey(pElem), nTerm);
pReader->zTerm[nTerm] = '\0';
pReader->nTerm = nTerm;
aCopy = (char*)sqlite3_malloc64(nCopy);
pReader->zTerm = (char *)fts3HashKey(pElem);
pReader->nTerm = fts3HashKeysize(pElem);
aCopy = (char*)sqlite3_malloc(nCopy);
if( !aCopy ) return SQLITE_NOMEM;
memcpy(aCopy, pList->aData, nCopy);
pReader->nNode = pReader->nDoclist = nCopy;
@@ -1599,7 +1589,9 @@ int sqlite3Fts3MsrOvfl(
*/
void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
if( pReader ){
sqlite3_free(pReader->zTerm);
if( !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
}
if( !fts3SegReaderIsRootOnly(pReader) ){
sqlite3_free(pReader->aNode);
}
@@ -1634,7 +1626,7 @@ int sqlite3Fts3SegReaderNew(
nExtra = nRoot + FTS3_NODE_PADDING;
}
pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra);
pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
if( !pReader ){
return SQLITE_NOMEM;
}
@@ -1726,7 +1718,7 @@ int sqlite3Fts3SegReaderPending(
if( nElem==nAlloc ){
Fts3HashElem **aElem2;
nAlloc += 16;
aElem2 = (Fts3HashElem **)sqlite3_realloc64(
aElem2 = (Fts3HashElem **)sqlite3_realloc(
aElem, nAlloc*sizeof(Fts3HashElem *)
);
if( !aElem2 ){
@@ -2060,7 +2052,7 @@ static int fts3NodeAddTerm(
** this is not expected to be a serious problem.
*/
assert( pTree->aData==(char *)&pTree[1] );
pTree->aData = (char *)sqlite3_malloc64(nReq);
pTree->aData = (char *)sqlite3_malloc(nReq);
if( !pTree->aData ){
return SQLITE_NOMEM;
}
@@ -2078,7 +2070,7 @@ static int fts3NodeAddTerm(
if( isCopyTerm ){
if( pTree->nMalloc<nTerm ){
char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2);
char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2);
if( !zNew ){
return SQLITE_NOMEM;
}
@@ -2104,7 +2096,7 @@ static int fts3NodeAddTerm(
** now. Instead, the term is inserted into the parent of pTree. If pTree
** has no parent, one is created here.
*/
pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize);
pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize);
if( !pNew ){
return SQLITE_NOMEM;
}
@@ -2242,7 +2234,7 @@ static int fts3SegWriterAdd(
){
int nPrefix; /* Size of term prefix in bytes */
int nSuffix; /* Size of term suffix in bytes */
i64 nReq; /* Number of bytes required on leaf page */
int nReq; /* Number of bytes required on leaf page */
int nData;
SegmentWriter *pWriter = *ppWriter;
@@ -2251,13 +2243,13 @@ static int fts3SegWriterAdd(
sqlite3_stmt *pStmt;
/* Allocate the SegmentWriter structure */
pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter));
pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
if( !pWriter ) return SQLITE_NOMEM;
memset(pWriter, 0, sizeof(SegmentWriter));
*ppWriter = pWriter;
/* Allocate a buffer in which to accumulate data */
pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize);
pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize);
if( !pWriter->aData ) return SQLITE_NOMEM;
pWriter->nSize = p->nNodeSize;
@@ -2332,7 +2324,7 @@ static int fts3SegWriterAdd(
** the buffer to make it large enough.
*/
if( nReq>pWriter->nSize ){
char *aNew = sqlite3_realloc64(pWriter->aData, nReq);
char *aNew = sqlite3_realloc(pWriter->aData, nReq);
if( !aNew ) return SQLITE_NOMEM;
pWriter->aData = aNew;
pWriter->nSize = nReq;
@@ -2357,7 +2349,7 @@ static int fts3SegWriterAdd(
*/
if( isCopyTerm ){
if( nTerm>pWriter->nMalloc ){
char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2);
char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
if( !zNew ){
return SQLITE_NOMEM;
}
@@ -2665,20 +2657,18 @@ static void fts3ColumnFilter(
static int fts3MsrBufferData(
Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
char *pList,
i64 nList
int nList
){
if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){
if( nList>pMsr->nBuffer ){
char *pNew;
int nNew = nList*2 + FTS3_NODE_PADDING;
pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew);
pMsr->nBuffer = nList*2;
pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer);
if( !pNew ) return SQLITE_NOMEM;
pMsr->aBuffer = pNew;
pMsr->nBuffer = nNew;
}
assert( nList>0 );
memcpy(pMsr->aBuffer, pList, nList);
memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING);
return SQLITE_OK;
}
@@ -2728,7 +2718,7 @@ int sqlite3Fts3MsrIncrNext(
fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){
rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1);
rc = fts3MsrBufferData(pMsr, pList, nList+1);
if( rc!=SQLITE_OK ) return rc;
assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 );
pList = pMsr->aBuffer;
@@ -2865,11 +2855,11 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){
return SQLITE_OK;
}
static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){
static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){
if( nReq>pCsr->nBuffer ){
char *aNew;
pCsr->nBuffer = nReq*2;
aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer);
aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
if( !aNew ){
return SQLITE_NOMEM;
}
@@ -2960,8 +2950,7 @@ int sqlite3Fts3SegReaderStep(
){
pCsr->nDoclist = apSegment[0]->nDoclist;
if( fts3SegReaderIsPending(apSegment[0]) ){
rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist,
(i64)pCsr->nDoclist);
rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist);
pCsr->aDoclist = pCsr->aBuffer;
}else{
pCsr->aDoclist = apSegment[0]->aDoclist;
@@ -3014,8 +3003,7 @@ int sqlite3Fts3SegReaderStep(
nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
rc = fts3GrowSegReaderBuffer(pCsr,
(i64)nByte+nDoclist+FTS3_NODE_PADDING);
rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING);
if( rc ) return rc;
if( isFirst ){
@@ -3041,7 +3029,7 @@ int sqlite3Fts3SegReaderStep(
fts3SegReaderSort(apSegment, nMerge, j, xCmp);
}
if( nDoclist>0 ){
rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING);
rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING);
if( rc ) return rc;
memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING);
pCsr->aDoclist = pCsr->aBuffer;
@@ -3325,6 +3313,7 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
sqlite3Fts3PendingTermsClear(p);
/* Determine the auto-incr-merge setting if unknown. If enabled,
** estimate the number of leaf blocks of content to be written
@@ -3346,10 +3335,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
rc = sqlite3_reset(pStmt);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts3PendingTermsClear(p);
}
return rc;
}
@@ -3757,7 +3742,7 @@ struct NodeReader {
static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){
if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){
int nAlloc = nMin;
char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc);
char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc);
if( a ){
pBlob->nAlloc = nAlloc;
pBlob->a = a;
@@ -3798,7 +3783,7 @@ static int nodeReaderNext(NodeReader *p){
return FTS_CORRUPT_VTAB;
}
blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
if( rc==SQLITE_OK && ALWAYS(p->term.a!=0) ){
if( rc==SQLITE_OK ){
memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
p->term.n = nPrefix+nSuffix;
p->iOff += nSuffix;
@@ -3906,8 +3891,6 @@ static int fts3IncrmergePush(
pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix);
}
pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix);
assert( nPrefix+nSuffix<=nTerm );
assert( nPrefix>=0 );
memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix);
pBlk->n += nSuffix;
@@ -3981,8 +3964,6 @@ static int fts3AppendToNode(
blobGrowBuffer(pPrev, nTerm, &rc);
if( rc!=SQLITE_OK ) return rc;
assert( pPrev!=0 );
assert( pPrev->a!=0 );
nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
nSuffix = nTerm - nPrefix;
@@ -4032,20 +4013,15 @@ static int fts3IncrmergeAppend(
pLeaf = &pWriter->aNodeWriter[0];
nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm);
nSuffix = nTerm - nPrefix;
if(nSuffix<=0 ) return FTS_CORRUPT_VTAB;
nSpace = sqlite3Fts3VarintLen(nPrefix);
nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
/* If the current block is not empty, and if adding this term/doclist
** to the current block would make it larger than Fts3Table.nNodeSize bytes,
** and if there is still room for another leaf page, write this block out to
** the database. */
if( pLeaf->block.n>0
&& (pLeaf->block.n + nSpace)>p->nNodeSize
&& pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst)
){
** to the current block would make it larger than Fts3Table.nNodeSize
** bytes, write this block out to the database. */
if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
pWriter->nWork++;
@@ -4201,11 +4177,7 @@ static int fts3TermCmp(
int nCmp = MIN(nLhs, nRhs);
int res;
if( nCmp && ALWAYS(zLhs) && ALWAYS(zRhs) ){
res = memcmp(zLhs, zRhs, nCmp);
}else{
res = 0;
}
res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0);
if( res==0 ) res = nLhs - nRhs;
return res;
@@ -4356,7 +4328,6 @@ static int fts3IncrmergeLoad(
for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
NodeReader reader;
memset(&reader, 0, sizeof(reader));
pNode = &pWriter->aNodeWriter[i];
if( pNode->block.a){
@@ -4377,7 +4348,7 @@ static int fts3IncrmergeLoad(
rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0);
blobGrowBuffer(&pNode->block,
MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
);
);
if( rc==SQLITE_OK ){
memcpy(pNode->block.a, aBlock, nBlock);
pNode->block.n = nBlock;
@@ -4561,7 +4532,7 @@ static int fts3RepackSegdirLevel(
if( nIdx>=nAlloc ){
int *aNew;
nAlloc += 16;
aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int));
aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int));
if( !aNew ){
rc = SQLITE_NOMEM;
break;
@@ -4850,7 +4821,7 @@ static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){
if( aHint ){
blobGrowBuffer(pHint, nHint, &rc);
if( rc==SQLITE_OK ){
if( ALWAYS(pHint->a!=0) ) memcpy(pHint->a, aHint, nHint);
memcpy(pHint->a, aHint, nHint);
pHint->n = nHint;
}
}
@@ -4935,7 +4906,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
/* Allocate space for the cursor, filter and writer objects */
const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);
pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc);
pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
if( !pWriter ) return SQLITE_NOMEM;
pFilter = (Fts3SegFilter *)&pWriter[1];
pCsr = (Fts3MultiSegReader *)&pFilter[1];
@@ -5227,7 +5198,7 @@ static u64 fts3ChecksumIndex(
int rc;
u64 cksum = 0;
if( *pRc ) return 0;
assert( *pRc==SQLITE_OK );
memset(&filter, 0, sizeof(filter));
memset(&csr, 0, sizeof(csr));
@@ -5294,7 +5265,7 @@ static u64 fts3ChecksumIndex(
** If an error occurs (e.g. an OOM or IO error), return an SQLite error
** code. The final value of *pbOk is undefined in this case.
*/
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int rc = SQLITE_OK; /* Return code */
u64 cksum1 = 0; /* Checksum based on FTS index contents */
u64 cksum2 = 0; /* Checksum based on %_content contents */
@@ -5372,12 +5343,7 @@ int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
sqlite3_finalize(pStmt);
}
if( rc==SQLITE_CORRUPT_VTAB ){
rc = SQLITE_OK;
*pbOk = 0;
}else{
*pbOk = (rc==SQLITE_OK && cksum1==cksum2);
}
*pbOk = (cksum1==cksum2);
return rc;
}
@@ -5417,7 +5383,7 @@ static int fts3DoIntegrityCheck(
){
int rc;
int bOk = 0;
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
rc = fts3IntegrityCheck(p, &bOk);
if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
return rc;
}
@@ -5447,11 +5413,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
rc = fts3DoIncrmerge(p, &zVal[6]);
}else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
rc = fts3DoAutoincrmerge(p, &zVal[10]);
}else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
rc = sqlite3Fts3PendingTermsFlush(p);
}
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
else{
}else{
int v;
if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
v = atoi(&zVal[9]);
@@ -5469,8 +5432,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
rc = SQLITE_OK;
}
}
#endif
}
return rc;
}
@@ -5579,7 +5542,7 @@ int sqlite3Fts3DeferredTokenList(
return SQLITE_OK;
}
pRet = (char *)sqlite3_malloc64(p->pList->nData);
pRet = (char *)sqlite3_malloc(p->pList->nData);
if( !pRet ) return SQLITE_NOMEM;
nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
@@ -5599,7 +5562,7 @@ int sqlite3Fts3DeferToken(
int iCol /* Column that token must appear in (or -1) */
){
Fts3DeferredToken *pDeferred;
pDeferred = sqlite3_malloc64(sizeof(*pDeferred));
pDeferred = sqlite3_malloc(sizeof(*pDeferred));
if( !pDeferred ){
return SQLITE_NOMEM;
}
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/tclsh
#
# This script builds a single C code file holding all of FTS3 code.
# The name of the output file is fts3amal.c. To build this file,
# first do:
#
# make target_source
#
# The make target above moves all of the source code files into
# a subdirectory named "tsrc". (This script expects to find the files
# there and will not work if they are not found.)
#
# After the "tsrc" directory has been created and populated, run
# this script:
#
# tclsh mkfts3amal.tcl
#
# The amalgamated FTS3 code will be written into fts3amal.c
#
# Open the output file and write a header comment at the beginning
# of the file.
#
set out [open fts3amal.c w]
set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
puts $out [subst \
{/******************************************************************************
** This file is an amalgamation of separate C source files from the SQLite
** Full Text Search extension 2 (fts3). By combining all the individual C
** code files into this single large file, the entire code can be compiled
** as a one translation unit. This allows many compilers to do optimizations
** that would not be possible if the files were compiled separately. It also
** makes the code easier to import into other projects.
**
** This amalgamation was generated on $today.
*/}]
# These are the header files used by FTS3. The first time any of these
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line. The file only needs to be included once.
#
foreach hdr {
fts3.h
fts3_hash.h
fts3_tokenizer.h
sqlite3.h
sqlite3ext.h
} {
set available_hdr($hdr) 1
}
# 78 stars used for comment formatting.
set s78 \
{*****************************************************************************}
# Insert a comment into the code
#
proc section_comment {text} {
global out s78
set n [string length $text]
set nstar [expr {60 - $n}]
set stars [string range $s78 0 $nstar]
puts $out "/************** $text $stars/"
}
# Read the source file named $filename and write it into the
# sqlite3.c output file. If any #include statements are seen,
# process them approprately.
#
proc copy_file {filename} {
global seen_hdr available_hdr out
set tail [file tail $filename]
section_comment "Begin file $tail"
set in [open $filename r]
while {![eof $in]} {
set line [gets $in]
if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
if {[info exists available_hdr($hdr)]} {
if {$available_hdr($hdr)} {
section_comment "Include $hdr in the middle of $tail"
copy_file tsrc/$hdr
section_comment "Continuing where we left off in $tail"
}
} elseif {![info exists seen_hdr($hdr)]} {
set seen_hdr($hdr) 1
puts $out $line
}
} elseif {[regexp {^#ifdef __cplusplus} $line]} {
puts $out "#if 0"
} elseif {[regexp {^#line} $line]} {
# Skip #line directives.
} else {
puts $out $line
}
}
close $in
section_comment "End of $tail"
}
# Process the source files. Process files containing commonly
# used subroutines first in order to help the compiler find
# inlining opportunities.
#
foreach file {
fts3.c
fts3_hash.c
fts3_porter.c
fts3_tokenizer.c
fts3_tokenizer1.c
} {
copy_file tsrc/$file
}
close $out
-3
View File
@@ -628,9 +628,6 @@ proc print_categories {lMap} {
$caseP
$caseS
$caseZ
default:
return 1;
}
return 0;
}
+3 -8
View File
@@ -82,7 +82,7 @@ proc get_struct_docs {data names} {
set current_doc ""
}
set subject n/a
regexp {^ *([[:alnum:]_]*)} $line -> subject
regexp {^ *([[:alpha:]]*)} $line -> subject
if {[lsearch $names $subject]>=0} {
set current_header $subject
} else {
@@ -108,11 +108,8 @@ proc get_tokenizer_docs {data} {
append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
continue
}
if {[regexp {FTS5_TOKENIZER} $line]} {
set line </dl><p>
}
if {[regexp {SYNONYM SUPPORT} $line]} {
set line "<h3>Synonym Support</h3>"
set line "</dl><h3>Synonym Support</h3>"
}
if {[string trim $line] == ""} {
append res "<p>\n"
@@ -226,12 +223,10 @@ proc main {data} {
Fts5ExtensionApi {
set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
set map [list]
set lKey [list]
foreach {k v} [get_struct_members $data] {
if {[string match x* $k]==0} continue
lappend lKey $k
lappend map $k "<a href=#$k>$k</a>"
}
foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
output [string map $map $struct]
}
+19 -184
View File
@@ -55,8 +55,8 @@ struct Fts5PhraseIter {
** EXTENSION API FUNCTIONS
**
** xUserData(pFts):
** Return a copy of the pUserData pointer passed to the xCreateFunction()
** API when the extension function was registered.
** Return a copy of the context pointer the extension function was
** registered with.
**
** xColumnTotalSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
@@ -88,11 +88,8 @@ struct Fts5PhraseIter {
** created with the "columnsize=0" option.
**
** xColumnText:
** If parameter iCol is less than zero, or greater than or equal to the
** number of columns in the table, SQLITE_RANGE is returned.
**
** Otherwise, this function attempts to retrieve the text of column iCol of
** the current document. If successful, (*pz) is set to point to a buffer
** This function attempts to retrieve the text of column iCol of the
** current document. If successful, (*pz) is set to point to a buffer
** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
** if an error occurs, an SQLite error code is returned and the final values
@@ -102,10 +99,8 @@ struct Fts5PhraseIter {
** Returns the number of phrases in the current query expression.
**
** xPhraseSize:
** If parameter iCol is less than zero, or greater than or equal to the
** number of phrases in the current query, as returned by xPhraseCount,
** 0 is returned. Otherwise, this function returns the number of tokens in
** phrase iPhrase of the query. Phrases are numbered starting from zero.
** Returns the number of tokens in phrase iPhrase of the query. Phrases
** are numbered starting from zero.
**
** xInstCount:
** Set *pnInst to the total number of occurrences of all phrases within
@@ -121,13 +116,12 @@ struct Fts5PhraseIter {
** Query for the details of phrase match iIdx within the current row.
** Phrase matches are numbered starting from zero, so the iIdx argument
** should be greater than or equal to zero and smaller than the value
** output by xInstCount(). If iIdx is less than zero or greater than
** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
** output by xInstCount().
**
** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
** Usually, output parameter *piPhrase is set to the phrase number, *piCol
** to the column in which it occurs and *piOff the token offset of the
** first token of the phrase. SQLITE_OK is returned if successful, or an
** error code (i.e. SQLITE_NOMEM) if an error occurs.
** first token of the phrase. Returns SQLITE_OK if successful, or an error
** code (i.e. SQLITE_NOMEM) if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
@@ -153,10 +147,6 @@ struct Fts5PhraseIter {
** Invoking Api.xUserData() returns a copy of the pointer passed as
** the third argument to pUserData.
**
** If parameter iPhrase is less than zero, or greater than or equal to
** the number of phrases in the query, as returned by xPhraseCount(),
** this function returns SQLITE_RANGE.
**
** If the callback function returns any value other than SQLITE_OK, the
** query is abandoned and the xQueryPhrase function returns immediately.
** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
@@ -238,10 +228,6 @@ struct Fts5PhraseIter {
** (i.e. if it is a contentless table), then this API always iterates
** through an empty set (all calls to xPhraseFirst() set iCol to -1).
**
** In all cases, matches are visited in (column ASC, offset ASC) order.
** i.e. all those in column 0, sorted by offset, followed by those in
** column 1, etc.
**
** xPhraseNext()
** See xPhraseFirst above.
**
@@ -275,65 +261,9 @@ struct Fts5PhraseIter {
**
** xPhraseNextColumn()
** See xPhraseFirstColumn above.
**
** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
** This is used to access token iToken of phrase iPhrase of the current
** query. Before returning, output parameter *ppToken is set to point
** to a buffer containing the requested token, and *pnToken to the
** size of this buffer in bytes.
**
** If iPhrase or iToken are less than zero, or if iPhrase is greater than
** or equal to the number of phrases in the query as reported by
** xPhraseCount(), or if iToken is equal to or greater than the number of
** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
are both zeroed.
**
** The output text is not a copy of the query text that specified the
** token. It is the output of the tokenizer module. For tokendata=1
** tables, this includes any embedded 0x00 and trailing data.
**
** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
** This is used to access token iToken of phrase hit iIdx within the
** current row. If iIdx is less than zero or greater than or equal to the
** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
** output variable (*ppToken) is set to point to a buffer containing the
** matching document token, and (*pnToken) to the size of that buffer in
** bytes. This API is not available if the specified token matches a
** prefix query term. In that case both output variables are always set
** to 0.
**
** The output text is not a copy of the document text that was tokenized.
** It is the output of the tokenizer module. For tokendata=1 tables, this
** includes any embedded 0x00 and trailing data.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
**
** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
** If parameter iCol is less than zero, or greater than or equal to the
** number of columns in the table, SQLITE_RANGE is returned.
**
** Otherwise, this function attempts to retrieve the locale associated
** with column iCol of the current row. Usually, there is no associated
** locale, and output parameters (*pzLocale) and (*pnLocale) are set
** to NULL and 0, respectively. However, if the fts5_locale() function
** was used to associate a locale with the value when it was inserted
** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
** buffer containing the name of the locale in utf-8 encoding. (*pnLocale)
** is set to the size in bytes of the buffer, not including the
** nul-terminator.
**
** If successful, SQLITE_OK is returned. Or, if an error occurs, an
** SQLite error code is returned. The final value of the output parameters
** is undefined in this case.
**
** xTokenize_v2:
** Tokenize text using the tokenizer belonging to the FTS5 table. This
** API is the same as the xTokenize() API, except that it allows a tokenizer
** locale to be specified.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 4 */
int iVersion; /* Currently always set to 3 */
void *(*xUserData)(Fts5Context*);
@@ -368,22 +298,6 @@ struct Fts5ExtensionApi {
int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
/* Below this point are iVersion>=3 only */
int (*xQueryToken)(Fts5Context*,
int iPhrase, int iToken,
const char **ppToken, int *pnToken
);
int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
/* Below this point are iVersion>=4 only */
int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xTokenize_v2)(Fts5Context*,
const char *pText, int nText, /* Text to tokenize */
const char *pLocale, int nLocale, /* Locale to pass to tokenizer */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
};
/*
@@ -396,7 +310,6 @@ struct Fts5ExtensionApi {
** Applications may also register custom tokenizer types. A tokenizer
** is registered by providing fts5 with a populated instance of the
** following structure. All structure methods must be defined, setting
**
** any member of the fts5_tokenizer struct to NULL leads to undefined
** behaviour. The structure methods are expected to function as follows:
**
@@ -405,7 +318,7 @@ struct Fts5ExtensionApi {
** A tokenizer instance is required to actually tokenize text.
**
** The first argument passed to this function is a copy of the (void*)
** pointer provided by the application when the fts5_tokenizer_v2 object
** pointer provided by the application when the fts5_tokenizer object
** was registered with FTS5 (the third argument to xCreateTokenizer()).
** The second and third arguments are an array of nul-terminated strings
** containing the tokenizer arguments, if any, specified following the
@@ -429,7 +342,7 @@ struct Fts5ExtensionApi {
** argument passed to this function is a pointer to an Fts5Tokenizer object
** returned by an earlier call to xCreate().
**
** The third argument indicates the reason that FTS5 is requesting
** The second argument indicates the reason that FTS5 is requesting
** tokenization of the supplied text. This is always one of the following
** four values:
**
@@ -453,13 +366,6 @@ struct Fts5ExtensionApi {
** on a columnsize=0 database.
** </ul>
**
** The sixth and seventh arguments passed to xTokenize() - pLocale and
** nLocale - are a pointer to a buffer containing the locale to use for
** tokenization (e.g. "en_US") and its size in bytes, respectively. The
** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
** which case nLocale is always 0) to indicate that the tokenizer should
** use its default locale.
**
** For each token in the input string, the supplied callback xToken() must
** be invoked. The first argument to it should be a copy of the pointer
** passed as the second argument to xTokenize(). The third and fourth
@@ -483,30 +389,6 @@ struct Fts5ExtensionApi {
** may abandon the tokenization and return any error code other than
** SQLITE_OK or SQLITE_DONE.
**
** If the tokenizer is registered using an fts5_tokenizer_v2 object,
** then the xTokenize() method has two additional arguments - pLocale
** and nLocale. These specify the locale that the tokenizer should use
** for the current request. If pLocale and nLocale are both 0, then the
** tokenizer should use its default locale. Otherwise, pLocale points to
** an nLocale byte buffer containing the name of the locale to use as utf-8
** text. pLocale is not nul-terminated.
**
** FTS5_TOKENIZER
**
** There is also an fts5_tokenizer object. This is an older, deprecated,
** version of fts5_tokenizer_v2. It is similar except that:
**
** <ul>
** <li> There is no "iVersion" field, and
** <li> The xTokenize() method does not take a locale argument.
** </ul>
**
** Legacy fts5_tokenizer tokenizers must be registered using the
** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
**
** Tokenizer implementations registered using either API may be retrieved
** using both xFindTokenizer() and xFindTokenizer_v2().
**
** SYNONYM SUPPORT
**
** Custom tokenizers may also support synonyms. Consider a case in which a
@@ -610,38 +492,11 @@ struct Fts5ExtensionApi {
** as separate queries of the FTS index are required for each synonym.
**
** When using methods (2) or (3), it is important that the tokenizer only
** provide synonyms when tokenizing document text (method (3)) or query
** text (method (2)), not both. Doing so will not cause any errors, but is
** provide synonyms when tokenizing document text (method (2)) or query
** text (method (3)), not both. Doing so will not cause any errors, but is
** inefficient.
*/
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
struct fts5_tokenizer_v2 {
int iVersion; /* Currently always 2 */
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
void (*xDelete)(Fts5Tokenizer*);
int (*xTokenize)(Fts5Tokenizer*,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
const char *pLocale, int nLocale,
int (*xToken)(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
)
);
};
/*
** New code should use the fts5_tokenizer_v2 type to define tokenizer
** implementations. The following type is included for legacy applications
** that still use it.
*/
typedef struct fts5_tokenizer fts5_tokenizer;
struct fts5_tokenizer {
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
@@ -661,7 +516,6 @@ struct fts5_tokenizer {
);
};
/* Flags that may be passed as the third argument to xTokenize() */
#define FTS5_TOKENIZE_QUERY 0x0001
#define FTS5_TOKENIZE_PREFIX 0x0002
@@ -681,13 +535,13 @@ struct fts5_tokenizer {
*/
typedef struct fts5_api fts5_api;
struct fts5_api {
int iVersion; /* Currently always set to 3 */
int iVersion; /* Currently always set to 2 */
/* Create a new tokenizer */
int (*xCreateTokenizer)(
fts5_api *pApi,
const char *zName,
void *pUserData,
void *pContext,
fts5_tokenizer *pTokenizer,
void (*xDestroy)(void*)
);
@@ -696,7 +550,7 @@ struct fts5_api {
int (*xFindTokenizer)(
fts5_api *pApi,
const char *zName,
void **ppUserData,
void **ppContext,
fts5_tokenizer *pTokenizer
);
@@ -704,29 +558,10 @@ struct fts5_api {
int (*xCreateFunction)(
fts5_api *pApi,
const char *zName,
void *pUserData,
void *pContext,
fts5_extension_function xFunction,
void (*xDestroy)(void*)
);
/* APIs below this point are only available if iVersion>=3 */
/* Create a new tokenizer */
int (*xCreateTokenizer_v2)(
fts5_api *pApi,
const char *zName,
void *pUserData,
fts5_tokenizer_v2 *pTokenizer,
void (*xDestroy)(void*)
);
/* Find an existing tokenizer */
int (*xFindTokenizer_v2)(
fts5_api *pApi,
const char *zName,
void **ppUserData,
fts5_tokenizer_v2 **ppTokenizer
);
};
/*
+25 -116
View File
@@ -35,20 +35,8 @@ typedef sqlite3_uint64 u64;
#endif
#define testcase(x)
#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
#endif
#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
# define ALWAYS(X) (1)
# define NEVER(X) (0)
#elif !defined(NDEBUG)
# define ALWAYS(X) ((X)?1:(assert(0),0))
# define NEVER(X) ((X)?(assert(0),1):0)
#else
# define ALWAYS(X) (X)
# define NEVER(X) (X)
#endif
#define ALWAYS(x) 1
#define NEVER(x) 0
#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#define MAX(x,y) (((x) > (y)) ? (x) : (y))
@@ -59,22 +47,6 @@ typedef sqlite3_uint64 u64;
# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
/* The uptr type is an unsigned integer large enough to hold a pointer
*/
#if defined(HAVE_STDINT_H)
typedef uintptr_t uptr;
#elif SQLITE_PTRSIZE==4
typedef u32 uptr;
#else
typedef u64 uptr;
#endif
#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0)
#else
# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0)
#endif
#endif
/* Truncate very long tokens to this many bytes. Hard limit is
@@ -124,7 +96,7 @@ extern int sqlite3_fts5_may_be_corrupt;
** A version of memcmp() that does not cause asan errors if one of the pointer
** parameters is NULL and the number of bytes to compare is zero.
*/
#define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
#define fts5Memcmp(s1, s2, n) ((n)==0 ? 0 : memcmp((s1), (s2), (n)))
/* Mark a function parameter as unused, to suppress nuisance compiler
** warnings. */
@@ -158,18 +130,6 @@ struct Fts5Colset {
*/
typedef struct Fts5Config Fts5Config;
typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
struct Fts5TokenizerConfig {
Fts5Tokenizer *pTok;
fts5_tokenizer_v2 *pApi2;
fts5_tokenizer *pApi1;
const char **azArg;
int nArg;
int ePattern; /* FTS_PATTERN_XXX constant */
const char *pLocale; /* Current locale to use */
int nLocale; /* Size of pLocale in bytes */
};
/*
** An instance of the following structure encodes all information that can
@@ -182,10 +142,6 @@ struct Fts5TokenizerConfig {
** attempt to merge together. A value of 1 sets the object to use the
** compile time default. Zero disables auto-merge altogether.
**
** bContentlessDelete:
** True if the contentless_delete option was present in the CREATE
** VIRTUAL TABLE statement.
**
** zContent:
**
** zContentRowid:
@@ -209,12 +165,9 @@ struct Fts5TokenizerConfig {
**
** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
**
** bLocale:
** Set to true if locale=1 was specified when the table was created.
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
Fts5Global *pGlobal; /* Global fts5 object for handle db */
char *zDb; /* Database holding FTS index (e.g. "main") */
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
@@ -223,20 +176,17 @@ struct Fts5Config {
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
int eContent; /* An FTS5_CONTENT value */
int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */
char *zContent; /* content table */
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int bTokendata; /* "tokendata=" option value (dflt==0) */
int bLocale; /* "locale=" option value (dflt==0) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5TokenizerConfig t;
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
int bLock; /* True when table is preparing statement */
int ePattern; /* FTS_PATTERN_XXX constant */
/* Values loaded from the %_config table */
int iVersion; /* fts5 file format 'version' */
int iCookie; /* Incremented when %_config is modified */
int pgsz; /* Approximate page size used in %_data */
int nAutomerge; /* 'automerge' setting */
@@ -245,8 +195,6 @@ struct Fts5Config {
int nHashSize; /* Bytes of memory for in-memory hash */
char *zRank; /* Name of rank function */
char *zRankArgs; /* Arguments to rank function */
int bSecureDelete; /* 'secure-delete' */
int nDeleteMerge; /* 'deletemerge' */
/* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
char **pzErrmsg;
@@ -256,11 +204,8 @@ struct Fts5Config {
#endif
};
/* Current expected value of %_config table 'version' field. And
** the expected version if the 'secure-delete' option has ever been
** set on the table. */
#define FTS5_CURRENT_VERSION 4
#define FTS5_CURRENT_VERSION_SECUREDELETE 5
/* Current expected value of %_config table 'version' field */
#define FTS5_CURRENT_VERSION 4
#define FTS5_CONTENT_NORMAL 0
#define FTS5_CONTENT_NONE 1
@@ -299,8 +244,6 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
@@ -331,7 +274,7 @@ void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
@@ -345,7 +288,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);
#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
typedef struct Fts5PoslistReader Fts5PoslistReader;
@@ -418,19 +361,16 @@ struct Fts5IndexIter {
/*
** Values used as part of the flags argument passed to IndexQuery().
*/
#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
/* The following are used internally by the fts5_index.c module. They are
** defined here only to make it easier to avoid clashes with the flags
** above. */
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
#define FTS5INDEX_QUERY_SKIPHASH 0x0040
#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
#define FTS5INDEX_QUERY_SCANONETERM 0x0100
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
/*
** Create/destroy an Fts5Index object.
@@ -499,10 +439,6 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
void sqlite3Fts5StructureRelease(void*);
int sqlite3Fts5StructureTest(Fts5Index*, void*);
/*
** Used by xInstToken():
*/
int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
/*
** Insert or remove data to or from the index. Each time a document is
@@ -577,16 +513,6 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
/* Used to populate hash tables for xInstToken in detail=none/column mode. */
int sqlite3Fts5IndexIterWriteTokendata(
Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
@@ -599,7 +525,7 @@ int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint sqlite3Fts5GetVarint
#define fts5FastGetVarint32(a, iOff, nVal) { \
@@ -630,20 +556,18 @@ struct Fts5Table {
Fts5Index *pIndex; /* Full-text index */
};
int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
int sqlite3Fts5GetTokenizer(
Fts5Global*,
const char **azArg,
int nArg,
Fts5Config*,
char **pzErr
);
Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
int sqlite3Fts5FlushToDisk(Fts5Table*);
void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
);
/*
** End of interface to code in fts5.c.
**************************************************************************/
@@ -673,11 +597,6 @@ int sqlite3Fts5HashWrite(
*/
void sqlite3Fts5HashClear(Fts5Hash*);
/*
** Return true if the hash is empty, false otherwise.
*/
int sqlite3Fts5HashIsEmpty(Fts5Hash*);
int sqlite3Fts5HashQuery(
Fts5Hash*, /* Hash table to query */
int nPre,
@@ -694,13 +613,11 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
int sqlite3Fts5HashScanEof(Fts5Hash*);
void sqlite3Fts5HashScanEntry(Fts5Hash *,
const char **pzTerm, /* OUT: term (nul-terminated) */
int *pnTerm, /* OUT: Size of term in bytes */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
);
/*
** End of interface to code in fts5_hash.c.
**************************************************************************/
@@ -723,7 +640,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
@@ -749,9 +666,6 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);
void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/
@@ -824,10 +738,6 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
void sqlite3Fts5ExprClearTokens(Fts5Expr*);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
@@ -904,7 +814,6 @@ int sqlite3Fts5TokenizerPattern(
int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
Fts5Tokenizer *pTok
);
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/
+22 -129
View File
@@ -110,19 +110,15 @@ static int fts5CInstIterInit(
*/
typedef struct HighlightContext HighlightContext;
struct HighlightContext {
/* Constant parameters to fts5HighlightCb() */
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
int iRangeStart; /* First token to include */
int iRangeEnd; /* If non-zero, last token to include */
const char *zOpen; /* Opening highlight */
const char *zClose; /* Closing highlight */
const char *zIn; /* Input text */
int nIn; /* Size of input text in bytes */
/* Variables modified by fts5HighlightCb() */
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
int iOff; /* Have copied up to this offset in zIn[] */
int bOpen; /* True if highlight is open */
int iOff; /* Current offset within zIn[] */
char *zOut; /* Output value */
};
@@ -155,8 +151,8 @@ static int fts5HighlightCb(
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start byte offset of token */
int iEndOff /* End byte offset of token */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
@@ -167,66 +163,40 @@ static int fts5HighlightCb(
if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
iPos = p->iPos++;
if( p->iRangeEnd>=0 ){
if( p->iRangeEnd>0 ){
if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
}
/* If the parenthesis is open, and this token is not part of the current
** phrase, and the starting byte offset of this token is past the point
** that has currently been copied into the output buffer, close the
** parenthesis. */
if( p->bOpen
&& (iPos<=p->iter.iStart || p->iter.iStart<0)
&& iStartOff>p->iOff
){
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->bOpen = 0;
}
/* If this is the start of a new phrase, and the highlight is not open:
**
** * copy text from the input up to the start of the phrase, and
** * open the highlight.
*/
if( iPos==p->iter.iStart && p->bOpen==0 ){
if( iPos==p->iter.iStart ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->iOff = iStartOff;
p->bOpen = 1;
}
if( iPos==p->iter.iEnd ){
if( p->bOpen==0 ){
assert( p->iRangeEnd>=0 );
if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->bOpen = 1;
}
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->iOff = iEndOff;
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(&p->iter);
}
}
if( iPos==p->iRangeEnd ){
if( p->bOpen ){
if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
}
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->bOpen = 0;
}
if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
fts5HighlightAppend(&rc, p, p->zClose, -1);
}
}
return rc;
}
/*
** Implementation of highlight() function.
*/
@@ -251,28 +221,15 @@ static void fts5HighlightFunction(
memset(&ctx, 0, sizeof(HighlightContext));
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
ctx.iRangeEnd = -1;
rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
if( rc==SQLITE_RANGE ){
sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
rc = SQLITE_OK;
}else if( ctx.zIn ){
const char *pLoc = 0; /* Locale of column iCol */
int nLoc = 0; /* Size of pLoc in bytes */
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize_v2(
pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
@@ -449,7 +406,6 @@ static void fts5SnippetFunction(
iCol = sqlite3_value_int(apVal[0]);
ctx.zOpen = fts5ValueToText(apVal[1]);
ctx.zClose = fts5ValueToText(apVal[2]);
ctx.iRangeEnd = -1;
zEllips = fts5ValueToText(apVal[3]);
nToken = sqlite3_value_int(apVal[4]);
@@ -466,8 +422,6 @@ static void fts5SnippetFunction(
memset(&sFinder, 0, sizeof(Fts5SFinder));
for(i=0; i<nCol; i++){
if( iCol<0 || iCol==i ){
const char *pLoc = 0; /* Locale of column iCol */
int nLoc = 0; /* Size of pLoc in bytes */
int nDoc;
int nDocsize;
int ii;
@@ -475,10 +429,8 @@ static void fts5SnippetFunction(
sFinder.nFirst = 0;
rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xTokenize_v2(pFts,
sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
rc = pApi->xTokenize(pFts,
sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
);
if( rc!=SQLITE_OK ) break;
rc = pApi->xColumnSize(pFts, i, &nDocsize);
@@ -536,9 +488,6 @@ static void fts5SnippetFunction(
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( ctx.zIn ){
const char *pLoc = 0; /* Locale of column iBestCol */
int nLoc = 0; /* Bytes in pLoc */
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
}
@@ -557,15 +506,7 @@ static void fts5SnippetFunction(
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize_v2(
pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
if( ctx.iRangeEnd>=(nColSize-1) ){
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
@@ -746,53 +687,6 @@ static void fts5Bm25Function(
}
}
/*
** Implementation of fts5_get_locale() function.
*/
static void fts5GetLocaleFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
int iCol = 0;
int eType = 0;
int rc = SQLITE_OK;
const char *zLocale = 0;
int nLocale = 0;
/* xColumnLocale() must be available */
assert( pApi->iVersion>=4 );
if( nVal!=1 ){
const char *z = "wrong number of arguments to function fts5_get_locale()";
sqlite3_result_error(pCtx, z, -1);
return;
}
eType = sqlite3_value_numeric_type(apVal[0]);
if( eType!=SQLITE_INTEGER ){
const char *z = "non-integer argument passed to function fts5_get_locale()";
sqlite3_result_error(pCtx, z, -1);
return;
}
iCol = sqlite3_value_int(apVal[0]);
if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
sqlite3_result_error_code(pCtx, SQLITE_RANGE);
return;
}
rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
return;
}
sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT);
}
int sqlite3Fts5AuxInit(fts5_api *pApi){
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
@@ -800,10 +694,9 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
fts5_extension_function xFunc;/* Callback function */
void (*xDestroy)(void*); /* Destructor function */
} aBuiltin [] = {
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
{ "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
+2 -3
View File
@@ -66,9 +66,9 @@ void sqlite3Fts5BufferAppendBlob(
u32 nData,
const u8 *pData
){
assert_nc( *pRc || nData>=0 );
if( nData ){
if( fts5BufferGrow(pRc, pBuf, nData) ) return;
assert( pBuf->p!=0 );
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
@@ -170,7 +170,6 @@ int sqlite3Fts5PoslistNext64(
i64 *piOff /* IN/OUT: Current offset */
){
int i = *pi;
assert( a!=0 || i==0 );
if( i>=n ){
/* EOF */
*piOff = -1;
@@ -178,8 +177,8 @@ int sqlite3Fts5PoslistNext64(
}else{
i64 iOff = *piOff;
u32 iVal;
assert( a!=0 );
fts5FastGetVarint32(a, i, iVal);
assert( iVal>=0 );
if( iVal<=1 ){
if( iVal==0 ){
*pi = i;
+42 -160
View File
@@ -22,8 +22,6 @@
#define FTS5_DEFAULT_CRISISMERGE 16
#define FTS5_DEFAULT_HASHSIZE (1024*1024)
#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */
/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (64*1024)
@@ -234,6 +232,7 @@ static int fts5ConfigSetEnum(
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Global *pGlobal,
Fts5Config *pConfig, /* Configuration object to update */
const char *zCmd, /* Special command to parse */
const char *zArg, /* Argument to parse */
@@ -297,11 +296,12 @@ static int fts5ConfigParseSpecial(
if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
const char *p = (const char*)zArg;
sqlite3_int64 nArg = strlen(zArg) + 1;
char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
char *pSpace = pDel;
if( azArg ){
char *pSpace = (char*)&azArg[nArg];
if( pConfig->t.azArg ){
if( azArg && pSpace ){
if( pConfig->pTok ){
*pzErr = sqlite3_mprintf("multiple tokenize=... directives");
rc = SQLITE_ERROR;
}else{
@@ -324,14 +324,16 @@ static int fts5ConfigParseSpecial(
*pzErr = sqlite3_mprintf("parse error in tokenize directive");
rc = SQLITE_ERROR;
}else{
pConfig->t.azArg = (const char**)azArg;
pConfig->t.nArg = nArg;
azArg = 0;
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, (int)nArg, pConfig,
pzErr
);
}
}
}
sqlite3_free(azArg);
sqlite3_free(azArg);
sqlite3_free(pDel);
return rc;
}
@@ -350,16 +352,6 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bContentlessDelete = (zArg[0]=='1');
}
return rc;
}
if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
if( pConfig->zContentRowid ){
*pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -380,16 +372,6 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed locale=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bLocale = (zArg[0]=='1');
}
return rc;
}
if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
const Fts5Enum aDetail[] = {
{ "none", FTS5_DETAIL_NONE },
@@ -404,20 +386,20 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed tokendata=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bTokendata = (zArg[0]=='1');
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
return SQLITE_ERROR;
}
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
}
/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
@@ -516,15 +498,6 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){
}
}
}
if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
for(i=0; i<p->nCol; i++){
if( p->abUnindexed[i]==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
}else{
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
}
}
}
assert( p->zContentExprlist==0 );
p->zContentExprlist = (char*)buf.p;
@@ -559,7 +532,6 @@ int sqlite3Fts5ConfigParse(
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
memset(pRet, 0, sizeof(Fts5Config));
pRet->pGlobal = pGlobal;
pRet->db = db;
pRet->iCookie = -1;
@@ -578,7 +550,6 @@ int sqlite3Fts5ConfigParse(
rc = SQLITE_ERROR;
}
assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK );
for(i=3; rc==SQLITE_OK && i<nArg; i++){
const char *zOrig = azArg[i];
const char *z;
@@ -591,7 +562,6 @@ int sqlite3Fts5ConfigParse(
z = fts5ConfigSkipWhitespace(z);
if( z && *z=='=' ){
bOption = 1;
assert( zOne!=0 );
z++;
if( bMustBeCol ) z = 0;
}
@@ -608,11 +578,7 @@ int sqlite3Fts5ConfigParse(
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pRet,
ALWAYS(zOne)?zOne:"",
zTwo?zTwo:"",
pzErr
);
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
}else{
rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
zOne = 0;
@@ -624,26 +590,11 @@ int sqlite3Fts5ConfigParse(
sqlite3_free(zTwo);
}
/* We only allow contentless_delete=1 if the table is indeed contentless. */
if( rc==SQLITE_OK
&& pRet->bContentlessDelete
&& pRet->eContent!=FTS5_CONTENT_NONE
){
*pzErr = sqlite3_mprintf(
"contentless_delete=1 requires a contentless table"
);
rc = SQLITE_ERROR;
}
/* We only allow contentless_delete=1 if columnsize=0 is not present.
**
** This restriction may be removed at some point.
*/
if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
*pzErr = sqlite3_mprintf(
"contentless_delete=1 is incompatible with columnsize=0"
);
rc = SQLITE_ERROR;
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
@@ -687,14 +638,9 @@ int sqlite3Fts5ConfigParse(
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->t.pTok ){
if( pConfig->t.pApi1 ){
pConfig->t.pApi1->xDelete(pConfig->t.pTok);
}else{
pConfig->t.pApi2->xDelete(pConfig->t.pTok);
}
if( pConfig->pTok ){
pConfig->pTokApi->xDelete(pConfig->pTok);
}
sqlite3_free((char*)pConfig->t.azArg);
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
@@ -769,24 +715,10 @@ int sqlite3Fts5Tokenize(
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
){
int rc = SQLITE_OK;
if( pText ){
if( pConfig->t.pTok==0 ){
rc = sqlite3Fts5LoadTokenizer(pConfig);
}
if( rc==SQLITE_OK ){
if( pConfig->t.pApi1 ){
rc = pConfig->t.pApi1->xTokenize(
pConfig->t.pTok, pCtx, flags, pText, nText, xToken
);
}else{
rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags,
pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
);
}
}
}
return rc;
if( pText==0 ) return SQLITE_OK;
return pConfig->pTokApi->xTokenize(
pConfig->pTok, pCtx, flags, pText, nText, xToken
);
}
/*
@@ -952,18 +884,6 @@ int sqlite3Fts5ConfigSetValue(
}
}
else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
int nVal = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nVal = sqlite3_value_int(pVal);
}else{
*pbBadkey = 1;
}
if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
if( nVal>100 ) nVal = 0;
pConfig->nDeleteMerge = nVal;
}
else if( 0==sqlite3_stricmp(zKey, "rank") ){
const char *zIn = (const char*)sqlite3_value_text(pVal);
char *zRank;
@@ -978,18 +898,6 @@ int sqlite3Fts5ConfigSetValue(
rc = SQLITE_OK;
*pbBadkey = 1;
}
}
else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
int bVal = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
bVal = sqlite3_value_int(pVal);
}
if( bVal<0 ){
*pbBadkey = 1;
}else{
pConfig->bSecureDelete = (bVal ? 1 : 0);
}
}else{
*pbBadkey = 1;
}
@@ -1012,7 +920,6 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
if( zSql ){
@@ -1035,17 +942,15 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
rc = sqlite3_finalize(p);
}
if( rc==SQLITE_OK
&& iVersion!=FTS5_CURRENT_VERSION
&& iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
){
if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
rc = SQLITE_ERROR;
sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
"(found %d, expected %d or %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
);
}else{
pConfig->iVersion = iVersion;
if( pConfig->pzErrmsg ){
assert( 0==*pConfig->pzErrmsg );
*pConfig->pzErrmsg = sqlite3_mprintf(
"invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION
);
}
}
if( rc==SQLITE_OK ){
@@ -1053,26 +958,3 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
}
return rc;
}
/*
** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer
** containing the error message created using printf() style formatting
** string zFmt and its trailing arguments.
*/
void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
va_list ap; /* ... printf arguments */
char *zMsg = 0;
va_start(ap, zFmt);
zMsg = sqlite3_vmprintf(zFmt, ap);
if( pConfig->pzErrmsg ){
assert( *pConfig->pzErrmsg==0 );
*pConfig->pzErrmsg = zMsg;
}else{
sqlite3_free(zMsg);
}
va_end(ap);
}
+69 -278
View File
@@ -17,10 +17,6 @@
#include "fts5Int.h"
#include "fts5parse.h"
#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH
# define SQLITE_FTS5_MAX_EXPR_DEPTH 256
#endif
/*
** All token types in the generated fts5parse.h file are greater than 0.
*/
@@ -54,28 +50,18 @@ struct Fts5Expr {
/*
** eType:
** Expression node type. Usually one of:
** Expression node type. Always one of:
**
** FTS5_AND (nChild, apChild valid)
** FTS5_OR (nChild, apChild valid)
** FTS5_NOT (nChild, apChild valid)
** FTS5_STRING (pNear valid)
** FTS5_TERM (pNear valid)
**
** An expression node with eType==0 may also exist. It always matches zero
** rows. This is created when a phrase containing no tokens is parsed.
** e.g. "".
**
** iHeight:
** Distance from this node to furthest leaf. This is always 0 for nodes
** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one
** greater than the largest child value.
*/
struct Fts5ExprNode {
int eType; /* Node type */
int bEof; /* True at EOF */
int bNomatch; /* True if entry is not a match */
int iHeight; /* Distance to tree leaf nodes */
/* Next method for this node. */
int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
@@ -104,9 +90,7 @@ struct Fts5ExprNode {
struct Fts5ExprTerm {
u8 bPrefix; /* True for a prefix term */
u8 bFirst; /* True if token must be first in column */
char *pTerm; /* Term data */
int nQueryTerm; /* Effective size of term in bytes */
int nFullTerm; /* Size of term in bytes incl. tokendata */
char *zTerm; /* nul-terminated term */
Fts5IndexIter *pIter; /* Iterator for this term */
Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
};
@@ -147,31 +131,6 @@ struct Fts5Parse {
int bPhraseToAnd; /* Convert "a+b" to "a AND b" */
};
/*
** Check that the Fts5ExprNode.iHeight variables are set correctly in
** the expression tree passed as the only argument.
*/
#ifndef NDEBUG
static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){
if( rc==SQLITE_OK ){
if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){
assert( p->iHeight==0 );
}else{
int ii;
int iMaxChild = 0;
for(ii=0; ii<p->nChild; ii++){
Fts5ExprNode *pChild = p->apChild[ii];
iMaxChild = MAX(iMaxChild, pChild->iHeight);
assert_expr_depth_ok(SQLITE_OK, pChild);
}
assert( p->iHeight==iMaxChild+1 );
}
}
}
#else
# define assert_expr_depth_ok(rc, p)
#endif
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
va_list ap;
va_start(ap, zFmt);
@@ -286,12 +245,9 @@ int sqlite3Fts5ExprNew(
}while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
assert( sParse.pExpr || sParse.rc!=SQLITE_OK );
assert_expr_depth_ok(sParse.rc, sParse.pExpr);
/* If the LHS of the MATCH expression was a user column, apply the
** implicit column-filter. */
if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){
if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
int n = sizeof(Fts5Colset);
Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
if( pColset ){
@@ -308,7 +264,15 @@ int sqlite3Fts5ExprNew(
sParse.rc = SQLITE_NOMEM;
sqlite3Fts5ParseNodeFree(sParse.pExpr);
}else{
pNew->pRoot = sParse.pExpr;
if( !sParse.pExpr ){
const int nByte = sizeof(Fts5ExprNode);
pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
if( pNew->pRoot ){
pNew->pRoot->bEof = 1;
}
}else{
pNew->pRoot = sParse.pExpr;
}
pNew->pIndex = 0;
pNew->pConfig = pConfig;
pNew->apExprPhrase = sParse.apPhrase;
@@ -321,27 +285,10 @@ int sqlite3Fts5ExprNew(
}
sqlite3_free(sParse.apPhrase);
if( 0==*pzErr ){
*pzErr = sParse.zErr;
}else{
sqlite3_free(sParse.zErr);
}
*pzErr = sParse.zErr;
return sParse.rc;
}
/*
** Assuming that buffer z is at least nByte bytes in size and contains a
** valid utf-8 string, return the number of characters in the string.
*/
static int fts5ExprCountChar(const char *z, int nByte){
int nRet = 0;
int ii;
for(ii=0; ii<nByte; ii++){
if( (z[ii] & 0xC0)!=0x80 ) nRet++;
}
return nRet;
}
/*
** This function is only called when using the special 'trigram' tokenizer.
** Argument zText contains the text of a LIKE or GLOB pattern matched
@@ -379,8 +326,7 @@ int sqlite3Fts5ExprPattern(
if( i==nText
|| zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2]
){
if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){
if( i-iFirst>=3 ){
int jj;
zExpr[iOut++] = '"';
for(jj=iFirst; jj<i; jj++){
@@ -447,7 +393,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
Fts5Parse sParse;
memset(&sParse, 0, sizeof(sParse));
if( *pp1 && p2 ){
if( *pp1 ){
Fts5Expr *p1 = *pp1;
int nPhrase = p1->nPhrase + p2->nPhrase;
@@ -472,7 +418,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
}
sqlite3_free(p2->apExprPhrase);
sqlite3_free(p2);
}else if( p2 ){
}else{
*pp1 = p2;
}
@@ -488,7 +434,6 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
int bRetValid = 0;
Fts5ExprTerm *p;
assert( pTerm );
assert( pTerm->pSynonym );
assert( bDesc==0 || bDesc==1 );
for(p=pTerm; p; p=p->pSynonym){
@@ -970,7 +915,7 @@ static int fts5ExprNearInitAll(
p->pIter = 0;
}
rc = sqlite3Fts5IndexQuery(
pExpr->pIndex, p->pTerm, p->nQueryTerm,
pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
pNear->pColset,
@@ -1126,7 +1071,7 @@ static int fts5ExprNodeTest_STRING(
}
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
if( pIter->iRowid==iLast ) continue;
if( pIter->iRowid==iLast || pIter->bEof ) continue;
bMatch = 0;
if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
return rc;
@@ -1607,7 +1552,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
Fts5ExprTerm *pSyn;
Fts5ExprTerm *pNext;
Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
sqlite3_free(pTerm->pTerm);
sqlite3_free(pTerm->zTerm);
sqlite3Fts5IterClose(pTerm->pIter);
for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
pNext = pSyn->pSynonym;
@@ -1648,6 +1593,9 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5ExprNearset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
if( pPhrase==0 ){
return pNear;
}
if( pNear==0 ){
sqlite3_int64 nByte;
nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
@@ -1678,9 +1626,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
}else{
if( pRet->nPhrase>0 ){
Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
assert( pParse!=0 );
assert( pParse->apPhrase!=0 );
assert( pParse->nPhrase>=2 );
assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
if( pPhrase->nTerm==0 ){
fts5ExprPhraseFree(pPhrase);
@@ -1702,7 +1647,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
typedef struct TokenCtx TokenCtx;
struct TokenCtx {
Fts5ExprPhrase *pPhrase;
Fts5Config *pConfig;
int rc;
};
@@ -1736,12 +1680,8 @@ static int fts5ParseTokenize(
rc = SQLITE_NOMEM;
}else{
memset(pSyn, 0, (size_t)nByte);
pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
if( pCtx->pConfig->bTokendata ){
pSyn->nQueryTerm = (int)strlen(pSyn->pTerm);
}
memcpy(pSyn->pTerm, pToken, nToken);
pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
memcpy(pSyn->zTerm, pToken, nToken);
pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
}
@@ -1766,11 +1706,7 @@ static int fts5ParseTokenize(
if( rc==SQLITE_OK ){
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){
pTerm->nQueryTerm = (int)strlen(pTerm->pTerm);
}
pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
}
}
@@ -1837,7 +1773,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pAppend;
sCtx.pConfig = pConfig;
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
@@ -1869,7 +1804,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
}else if( sCtx.pPhrase->nTerm ){
sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
}
assert( pParse->apPhrase!=0 );
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
}
@@ -1886,15 +1820,12 @@ int sqlite3Fts5ExprClonePhrase(
Fts5Expr **ppNew
){
int rc = SQLITE_OK; /* Return code */
Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */
Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */
if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
rc = SQLITE_RANGE;
}else{
pOrig = pExpr->apExprPhrase[iPhrase];
pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
}
TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */
pOrig = pExpr->apExprPhrase[iPhrase];
pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
if( rc==SQLITE_OK ){
pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
sizeof(Fts5ExprPhrase*));
@@ -1907,7 +1838,7 @@ int sqlite3Fts5ExprClonePhrase(
pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
}
if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){
if( rc==SQLITE_OK ){
Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
if( pColsetOrig ){
sqlite3_int64 nByte;
@@ -1921,30 +1852,29 @@ int sqlite3Fts5ExprClonePhrase(
}
}
if( rc==SQLITE_OK ){
if( pOrig->nTerm ){
int i; /* Used to iterate through phrase terms */
sCtx.pConfig = pExpr->pConfig;
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
int tflags = 0;
Fts5ExprTerm *p;
for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
}
if( pOrig->nTerm ){
int i; /* Used to iterate through phrase terms */
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
int tflags = 0;
Fts5ExprTerm *p;
for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
const char *zTerm = p->zTerm;
rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
0, 0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
}
}else{
/* This happens when parsing a token or quoted phrase that contains
** no token characters at all. (e.g ... MATCH '""'). */
sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
}
}else{
/* This happens when parsing a token or quoted phrase that contains
** no token characters at all. (e.g ... MATCH '""'). */
sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
}
if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){
if( rc==SQLITE_OK ){
/* All the allocations succeeded. Put the expression object together. */
pNew->pIndex = pExpr->pIndex;
pNew->pConfig = pExpr->pConfig;
@@ -2257,11 +2187,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
}
}
/*
** Add pSub as a child of p.
*/
static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
int ii = p->nChild;
if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
@@ -2270,9 +2196,6 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
}else{
p->apChild[p->nChild++] = pSub;
}
for( ; ii<p->nChild; ii++){
p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1);
}
}
/*
@@ -2303,7 +2226,6 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
if( pRet ){
pRet->eType = FTS5_AND;
pRet->nChild = nTerm;
pRet->iHeight = 1;
fts5ExprAssignXNext(pRet);
pParse->nPhrase--;
for(ii=0; ii<nTerm; ii++){
@@ -2314,13 +2236,11 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
if( parseGrowPhraseArray(pParse) ){
fts5ExprPhraseFree(pPhrase);
}else{
Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
pParse->apPhrase[pParse->nPhrase++] = pPhrase;
pPhrase->nTerm = 1;
pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
pTo->nQueryTerm = p->nQueryTerm;
pTo->nFullTerm = p->nFullTerm;
pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
&pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
);
pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING,
0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
);
@@ -2404,25 +2324,13 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
"fts5: %s queries are not supported (detail!=full)",
pNear->nPhrase==1 ? "phrase": "NEAR"
);
sqlite3Fts5ParseNodeFree(pRet);
sqlite3_free(pRet);
pRet = 0;
pNear = 0;
assert( pLeft==0 && pRight==0 );
}
}
}else{
assert( pNear==0 );
fts5ExprAddChildren(pRet, pLeft);
fts5ExprAddChildren(pRet, pRight);
pLeft = pRight = 0;
if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
sqlite3Fts5ParseError(pParse,
"fts5 expression tree is too large (maximum depth %d)",
SQLITE_FTS5_MAX_EXPR_DEPTH
);
sqlite3Fts5ParseNodeFree(pRet);
pRet = 0;
}
}
}
}
@@ -2458,7 +2366,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
assert( pRight->eType==FTS5_STRING
|| pRight->eType==FTS5_TERM
|| pRight->eType==FTS5_EOF
|| (pRight->eType==FTS5_AND && pParse->bPhraseToAnd)
);
if( pLeft->eType==FTS5_AND ){
@@ -2472,8 +2379,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
);
if( pRight->eType==FTS5_EOF ){
assert( pParse->apPhrase!=0 );
assert( pParse->nPhrase>0 );
assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
sqlite3Fts5ParseNodeFree(pRight);
pRet = pLeft;
@@ -2504,7 +2409,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
return pRet;
}
#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
#ifdef SQLITE_TEST
static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
sqlite3_int64 nByte = 0;
Fts5ExprTerm *p;
@@ -2512,17 +2417,16 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
/* Determine the maximum amount of space required. */
for(p=pTerm; p; p=p->pSynonym){
nByte += pTerm->nQueryTerm * 2 + 3 + 2;
nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
}
zQuoted = sqlite3_malloc64(nByte);
if( zQuoted ){
int i = 0;
for(p=pTerm; p; p=p->pSynonym){
char *zIn = p->pTerm;
char *zEnd = &zIn[p->nQueryTerm];
char *zIn = p->zTerm;
zQuoted[i++] = '"';
while( zIn<zEnd ){
while( *zIn ){
if( *zIn=='"' ) zQuoted[i++] = '"';
zQuoted[i++] = *zIn++;
}
@@ -2600,10 +2504,8 @@ static char *fts5ExprPrintTcl(
zRet = fts5PrintfAppend(zRet, " {");
for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ",
p->nQueryTerm, p->pTerm
);
char *zTerm = pPhrase->aTerm[iTerm].zTerm;
zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
if( pPhrase->aTerm[iTerm].bPrefix ){
zRet = fts5PrintfAppend(zRet, "*");
}
@@ -2613,8 +2515,6 @@ static char *fts5ExprPrintTcl(
if( zRet==0 ) return 0;
}
}else if( pExpr->eType==0 ){
zRet = sqlite3_mprintf("{}");
}else{
char const *zOp = 0;
int i;
@@ -2876,14 +2776,14 @@ static void fts5ExprFold(
sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
}
}
#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
#endif /* ifdef SQLITE_TEST */
/*
** This is called during initialization to register the fts5_expr() scalar
** UDF with the SQLite handle passed as the only argument.
*/
int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
#ifdef SQLITE_TEST
struct Fts5ExprFunc {
const char *z;
void (*x)(sqlite3_context*,int,sqlite3_value**);
@@ -2955,15 +2855,6 @@ struct Fts5PoslistPopulator {
int bMiss;
};
/*
** Clear the position lists associated with all phrases in the expression
** passed as the first argument. Argument bLive is true if the expression
** might be pointing to a real entry, otherwise it has just been reset.
**
** At present this function is only used for detail=col and detail=none
** fts5 tables. This implies that all phrases must be at most 1 token
** in size, as phrase matches are not supported without detail=full.
*/
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
Fts5PoslistPopulator *pRet;
pRet = sqlite3_malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
@@ -2973,7 +2864,7 @@ Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
for(i=0; i<pExpr->nPhrase; i++){
Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
assert( pExpr->apExprPhrase[i]->nTerm<=1 );
assert( pExpr->apExprPhrase[i]->nTerm==1 );
if( bLive &&
(pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
){
@@ -3004,17 +2895,6 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
return 0;
}
/*
** pToken is a buffer nToken bytes in size that may or may not contain
** an embedded 0x00 byte. If it does, return the number of bytes in
** the buffer before the 0x00. If it does not, return nToken.
*/
static int fts5QueryTerm(const char *pToken, int nToken){
int ii;
for(ii=0; ii<nToken && pToken[ii]; ii++){}
return ii;
}
static int fts5ExprPopulatePoslistsCb(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
@@ -3026,33 +2906,22 @@ static int fts5ExprPopulatePoslistsCb(
Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
Fts5Expr *pExpr = p->pExpr;
int i;
int nQuery = nToken;
i64 iRowid = pExpr->pRoot->iRowid;
UNUSED_PARAM2(iUnused1, iUnused2);
if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
if( pExpr->pConfig->bTokendata ){
nQuery = fts5QueryTerm(pToken, nQuery);
}
if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprTerm *pT;
Fts5ExprTerm *pTerm;
if( p->aPopulator[i].bOk==0 ) continue;
for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
&& memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
int nTerm = (int)strlen(pTerm->zTerm);
if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
&& memcmp(pTerm->zTerm, pToken, nTerm)==0
){
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
int iCol = p->iOff>>32;
int iTokOff = p->iOff & 0x7FFFFFFF;
rc = sqlite3Fts5IndexIterWriteTokendata(
pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
);
}
if( rc ) return rc;
break;
}
@@ -3106,7 +2975,6 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
pNode->iRowid = iRowid;
pNode->bEof = 0;
switch( pNode->eType ){
case 0:
case FTS5_TERM:
case FTS5_STRING:
return (pNode->pNear->apPhrase[0]->poslist.n>0);
@@ -3188,80 +3056,3 @@ int sqlite3Fts5ExprPhraseCollist(
return rc;
}
/*
** Does the work of the fts5_api.xQueryToken() API method.
*/
int sqlite3Fts5ExprQueryToken(
Fts5Expr *pExpr,
int iPhrase,
int iToken,
const char **ppOut,
int *pnOut
){
Fts5ExprPhrase *pPhrase = 0;
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
return SQLITE_RANGE;
}
pPhrase = pExpr->apExprPhrase[iPhrase];
if( iToken<0 || iToken>=pPhrase->nTerm ){
return SQLITE_RANGE;
}
*ppOut = pPhrase->aTerm[iToken].pTerm;
*pnOut = pPhrase->aTerm[iToken].nFullTerm;
return SQLITE_OK;
}
/*
** Does the work of the fts5_api.xInstToken() API method.
*/
int sqlite3Fts5ExprInstToken(
Fts5Expr *pExpr,
i64 iRowid,
int iPhrase,
int iCol,
int iOff,
int iToken,
const char **ppOut,
int *pnOut
){
Fts5ExprPhrase *pPhrase = 0;
Fts5ExprTerm *pTerm = 0;
int rc = SQLITE_OK;
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
return SQLITE_RANGE;
}
pPhrase = pExpr->apExprPhrase[iPhrase];
if( iToken<0 || iToken>=pPhrase->nTerm ){
return SQLITE_RANGE;
}
pTerm = &pPhrase->aTerm[iToken];
if( pTerm->bPrefix==0 ){
if( pExpr->pConfig->bTokendata ){
rc = sqlite3Fts5IterToken(
pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
);
}else{
*ppOut = pTerm->pTerm;
*pnOut = pTerm->nFullTerm;
}
}
return rc;
}
/*
** Clear the token mappings for all Fts5IndexIter objects mannaged by
** the expression passed as the only argument.
*/
void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
int ii;
for(ii=0; ii<pExpr->nPhrase; ii++){
Fts5ExprTerm *pT;
for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
sqlite3Fts5IndexIterClearTokendata(pT->pIter);
}
}
}
+25 -55
View File
@@ -36,15 +36,10 @@ struct Fts5Hash {
/*
** Each entry in the hash table is represented by an object of the
** following type. Each object, its key, and its current data are stored
** in a single memory allocation. The key immediately follows the object
** in memory. The position list data immediately follows the key data
** in memory.
**
** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
** byte identifying the index (either the main term index or a prefix-index),
** followed by the term data. For example: "0token". There is no
** nul-terminator - in this case nKey=6.
** following type. Each object, its key (a nul-terminated string) and
** its current data are stored in a single memory allocation. The
** key immediately follows the object in memory. The position list
** data immediately follows the key data in memory.
**
** The data that follows the key is in a similar, but not identical format
** to the doclist data stored in the database. It is:
@@ -179,7 +174,8 @@ static int fts5HashResize(Fts5Hash *pHash){
unsigned int iHash;
Fts5HashEntry *p = apOld[i];
apOld[i] = p->pHashNext;
iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
(int)strlen(fts5EntryKey(p)));
p->pHashNext = apNew[iHash];
apNew[iHash] = p;
}
@@ -263,7 +259,7 @@ int sqlite3Fts5HashWrite(
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
char *zKey = fts5EntryKey(p);
if( zKey[0]==bByte
&& p->nKey==nToken+1
&& p->nKey==nToken
&& memcmp(&zKey[1], pToken, nToken)==0
){
break;
@@ -293,9 +289,9 @@ int sqlite3Fts5HashWrite(
zKey[0] = bByte;
memcpy(&zKey[1], pToken, nToken);
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
p->nKey = nToken+1;
p->nKey = nToken;
zKey[nToken+1] = '\0';
p->nData = nToken+1 + sizeof(Fts5HashEntry);
p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
@@ -360,7 +356,7 @@ int sqlite3Fts5HashWrite(
p->bContent = 1;
}else{
/* Append a new column value, if necessary */
assert_nc( iCol>=p->iCol );
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
if( pHash->eDetail==FTS5_DETAIL_FULL ){
pPtr[p->nData++] = 0x01;
@@ -412,17 +408,12 @@ static Fts5HashEntry *fts5HashEntryMerge(
*ppOut = p1;
p1 = 0;
}else{
int i = 0;
char *zKey1 = fts5EntryKey(p1);
char *zKey2 = fts5EntryKey(p2);
int nMin = MIN(p1->nKey, p2->nKey);
while( zKey1[i]==zKey2[i] ) i++;
int cmp = memcmp(zKey1, zKey2, nMin);
if( cmp==0 ){
cmp = p1->nKey - p2->nKey;
}
assert( cmp!=0 );
if( cmp>0 ){
if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
/* p2 is smaller */
*ppOut = p2;
ppOut = &p2->pScanNext;
@@ -441,8 +432,10 @@ static Fts5HashEntry *fts5HashEntryMerge(
}
/*
** Link all tokens from hash table iHash into a list in sorted order. The
** tokens are not removed from the hash table.
** Extract all tokens from hash table iHash and link them into a list
** in sorted order. The hash table is cleared before returning. It is
** the responsibility of the caller to free the elements of the returned
** list.
*/
static int fts5HashEntrySort(
Fts5Hash *pHash,
@@ -464,7 +457,7 @@ static int fts5HashEntrySort(
Fts5HashEntry *pIter;
for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
if( pTerm==0
|| (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
|| (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
){
Fts5HashEntry *pEntry = pIter;
pEntry->pScanNext = 0;
@@ -482,6 +475,7 @@ static int fts5HashEntrySort(
pList = fts5HashEntryMerge(pList, ap[i]);
}
pHash->nEntry = 0;
sqlite3_free(ap);
*ppSorted = pList;
return SQLITE_OK;
@@ -503,11 +497,12 @@ int sqlite3Fts5HashQuery(
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
zKey = fts5EntryKey(p);
if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
assert( p->nKey+1==(int)strlen(zKey) );
if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
}
if( p ){
int nHashPre = sizeof(Fts5HashEntry) + nTerm;
int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
int nList = p->nData - nHashPre;
u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
if( pRet ){
@@ -534,28 +529,6 @@ int sqlite3Fts5HashScanInit(
return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
}
#ifdef SQLITE_DEBUG
static int fts5HashCount(Fts5Hash *pHash){
int nEntry = 0;
int ii;
for(ii=0; ii<pHash->nSlot; ii++){
Fts5HashEntry *p = 0;
for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
nEntry++;
}
}
return nEntry;
}
#endif
/*
** Return true if the hash table is empty, false otherwise.
*/
int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
assert( pHash->nEntry==fts5HashCount(pHash) );
return pHash->nEntry==0;
}
void sqlite3Fts5HashScanNext(Fts5Hash *p){
assert( !sqlite3Fts5HashScanEof(p) );
p->pScan = p->pScan->pScanNext;
@@ -568,22 +541,19 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
void sqlite3Fts5HashScanEntry(
Fts5Hash *pHash,
const char **pzTerm, /* OUT: term (nul-terminated) */
int *pnTerm, /* OUT: Size of term in bytes */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
){
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
char *zKey = fts5EntryKey(p);
int nTerm = p->nKey;
int nTerm = (int)strlen(zKey);
fts5HashAddPoslistSize(pHash, p, 0);
*pzTerm = zKey;
*pnTerm = nTerm;
*ppDoclist = (const u8*)&zKey[nTerm];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
*ppDoclist = (const u8*)&zKey[nTerm+1];
*pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
}else{
*pzTerm = 0;
*pnTerm = 0;
*ppDoclist = 0;
*pnDoclist = 0;
}
+364 -2690
View File
File diff suppressed because it is too large Load Diff
+205 -1048
View File
File diff suppressed because it is too large Load Diff
+94 -428
View File
@@ -16,40 +16,13 @@
#include "fts5Int.h"
/*
** pSavedRow:
** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it
** does a by-rowid lookup to retrieve a single row from the %_content
** table or equivalent external-content table/view.
**
** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
** values for a row being UPDATEd. In that case, the SQL statement is
** not reset and pSavedRow is set to point at it. This is so that the
** insert operation that follows the delete may access the original
** row values for any new values for which sqlite3_value_nochange() returns
** true. i.e. if the user executes:
**
** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
** ...
** UPDATE fts SET a=?, b=? WHERE rowid=?;
**
** then the value passed to the xUpdate() method of this table as the
** new.c value is an sqlite3_value_nochange() value. So in this case it
** must be read from the saved row stored in Fts5Storage.pSavedRow.
**
** This is necessary - using sqlite3_value_nochange() instead of just having
** SQLite pass the original value back via xUpdate() - so as not to discard
** any locale information associated with such values.
**
*/
struct Fts5Storage {
Fts5Config *pConfig;
Fts5Index *pIndex;
int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
i64 nTotalRow; /* Total number of rows in FTS table */
i64 *aTotalSize; /* Total sizes of each column */
sqlite3_stmt *pSavedRow;
sqlite3_stmt *aStmt[12];
sqlite3_stmt *aStmt[11];
};
@@ -63,39 +36,14 @@ struct Fts5Storage {
# error "FTS5_STMT_LOOKUP mismatch"
#endif
#define FTS5_STMT_LOOKUP2 3
#define FTS5_STMT_INSERT_CONTENT 4
#define FTS5_STMT_REPLACE_CONTENT 5
#define FTS5_STMT_DELETE_CONTENT 6
#define FTS5_STMT_REPLACE_DOCSIZE 7
#define FTS5_STMT_DELETE_DOCSIZE 8
#define FTS5_STMT_LOOKUP_DOCSIZE 9
#define FTS5_STMT_REPLACE_CONFIG 10
#define FTS5_STMT_SCAN 11
/*
** Return a pointer to a buffer obtained from sqlite3_malloc() that contains
** nBind comma-separated question marks. e.g. if nBind is passed 5, this
** function returns "?,?,?,?,?".
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op and
** NULL is returned immediately. Or, if the attempt to malloc a buffer
** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise,
** if it is SQLITE_OK when this function is called and the malloc() succeeds,
** *pRc is left unchanged.
*/
static char *fts5BindingsList(int *pRc, int nBind){
char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2);
if( zBind ){
int ii;
for(ii=0; ii<nBind; ii++){
zBind[ii*2] = '?';
zBind[ii*2 + 1] = ',';
}
zBind[ii*2-1] = '\0';
}
return zBind;
}
#define FTS5_STMT_INSERT_CONTENT 3
#define FTS5_STMT_REPLACE_CONTENT 4
#define FTS5_STMT_DELETE_CONTENT 5
#define FTS5_STMT_REPLACE_DOCSIZE 6
#define FTS5_STMT_DELETE_DOCSIZE 7
#define FTS5_STMT_LOOKUP_DOCSIZE 8
#define FTS5_STMT_REPLACE_CONFIG 9
#define FTS5_STMT_SCAN 10
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -125,15 +73,14 @@ static int fts5StorageGetStmt(
"SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
"SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
"SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
"SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
"DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
"REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */
"REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
"SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
"SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
"REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
"SELECT %s FROM %s AS T", /* SCAN */
@@ -141,8 +88,6 @@ static int fts5StorageGetStmt(
Fts5Config *pC = p->pConfig;
char *zSql = 0;
assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
switch( eStmt ){
case FTS5_STMT_SCAN:
zSql = sqlite3_mprintf(azStmt[eStmt],
@@ -159,45 +104,30 @@ static int fts5StorageGetStmt(
break;
case FTS5_STMT_LOOKUP:
case FTS5_STMT_LOOKUP2:
zSql = sqlite3_mprintf(azStmt[eStmt],
pC->zContentExprlist, pC->zContent, pC->zContentRowid
);
break;
case FTS5_STMT_INSERT_CONTENT: {
int nCol = 0;
case FTS5_STMT_INSERT_CONTENT:
case FTS5_STMT_REPLACE_CONTENT: {
int nCol = pC->nCol + 1;
char *zBind;
int i;
nCol = 1 + pC->nCol;
if( pC->bLocale ){
for(i=0; i<pC->nCol; i++){
if( pC->abUnindexed[i]==0 ) nCol++;
}
}
zBind = fts5BindingsList(&rc, nCol);
zBind = sqlite3_malloc64(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
}
zBind[i*2-1] = '\0';
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
sqlite3_free(zBind);
}
break;
}
case FTS5_STMT_REPLACE_DOCSIZE:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
(pC->bContentlessDelete ? ",?" : "")
);
break;
case FTS5_STMT_LOOKUP_DOCSIZE:
zSql = sqlite3_mprintf(azStmt[eStmt],
(pC->bContentlessDelete ? ",origin" : ""),
pC->zDb, pC->zName
);
break;
default:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
break;
@@ -207,7 +137,7 @@ static int fts5StorageGetStmt(
rc = SQLITE_NOMEM;
}else{
int f = SQLITE_PREPARE_PERSISTENT;
if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
p->pConfig->bLock++;
rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
p->pConfig->bLock--;
@@ -369,7 +299,7 @@ int sqlite3Fts5StorageOpen(
if( bCreate ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
int nDefn = 32 + pConfig->nCol*10;
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
@@ -381,25 +311,15 @@ int sqlite3Fts5StorageOpen(
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
if( pConfig->bLocale ){
for(i=0; i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i]==0 ){
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
}
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
sqlite3_free(zDefn);
}
if( rc==SQLITE_OK && pConfig->bColumnsize ){
const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
if( pConfig->bContentlessDelete ){
zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
}
rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
rc = sqlite3Fts5CreateTable(
pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
@@ -464,115 +384,53 @@ static int fts5StorageInsertCallback(
return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
}
/*
** This function is used as part of an UPDATE statement that modifies the
** rowid of a row. In that case, this function is called first to set
** Fts5Storage.pSavedRow to point to a statement that may be used to
** access the original values of the row being deleted - iDel.
**
** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
** It is not considered an error if row iDel does not exist. In this case
** pSavedRow is not set and SQLITE_OK returned.
*/
int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
int rc = SQLITE_OK;
sqlite3_stmt *pSeek = 0;
assert( p->pSavedRow==0 );
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
rc = sqlite3_reset(pSeek);
}else{
p->pSavedRow = pSeek;
}
}
return rc;
}
/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
**
** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
** the original values of the row being deleted. This is used by UPDATE
** statements.
*/
static int fts5StorageDeleteFromIndex(
Fts5Storage *p,
i64 iDel,
sqlite3_value **apVal,
int bSaveRow /* True to set pSavedRow */
sqlite3_value **apVal
){
Fts5Config *pConfig = p->pConfig;
sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
int rc = SQLITE_OK; /* Return code */
int rc; /* Return code */
int rc2; /* sqlite3_reset() return code */
int iCol;
Fts5InsertCtx ctx;
assert( bSaveRow==0 || apVal==0 );
assert( bSaveRow==0 || bSaveRow==1 );
assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
if( apVal==0 ){
if( p->pSavedRow && bSaveRow ){
pSeek = p->pSavedRow;
p->pSavedRow = 0;
}else{
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
return sqlite3_reset(pSeek);
}
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
return sqlite3_reset(pSeek);
}
}
ctx.pStorage = p;
ctx.iCol = -1;
rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
if( pConfig->abUnindexed[iCol-1]==0 ){
sqlite3_value *pVal = 0;
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
assert( pSeek==0 || apVal==0 );
assert( pSeek!=0 || apVal!=0 );
const char *zText;
int nText;
if( pSeek ){
pVal = sqlite3_column_value(pSeek, iCol);
zText = (const char*)sqlite3_column_text(pSeek, iCol);
nText = sqlite3_column_bytes(pSeek, iCol);
}else{
pVal = apVal[iCol-1];
zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
nText = sqlite3_value_bytes(apVal[iCol-1]);
}
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale && pSeek ){
pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol);
nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
pText, nText, (void*)&ctx, fts5StorageInsertCallback
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
}
sqlite3Fts5ClearLocale(pConfig);
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
zText, nText, (void*)&ctx, fts5StorageInsertCallback
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
if( p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
}
}
}
@@ -582,60 +440,11 @@ static int fts5StorageDeleteFromIndex(
p->nTotalRow--;
}
if( rc==SQLITE_OK && bSaveRow ){
assert( p->pSavedRow==0 );
p->pSavedRow = pSeek;
}else{
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
}
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
return rc;
}
/*
** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
** should be called by the xUpdate() method of the fts5 table before
** returning from any operation that may have set Fts5Storage.pSavedRow.
*/
void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
assert( pStorage->pSavedRow==0
|| pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2]
);
sqlite3_reset(pStorage->pSavedRow);
pStorage->pSavedRow = 0;
}
/*
** This function is called to process a DELETE on a contentless_delete=1
** table. It adds the tombstone required to delete the entry with rowid
** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
** an SQLite error code.
*/
static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
i64 iOrigin = 0;
sqlite3_stmt *pLookup = 0;
int rc = SQLITE_OK;
assert( p->pConfig->bContentlessDelete );
assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
/* Look up the origin of the document in the %_docsize table. Store
** this in stack variable iOrigin. */
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pLookup, 1, iDel);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
iOrigin = sqlite3_column_int64(pLookup, 1);
}
rc = sqlite3_reset(pLookup);
}
if( rc==SQLITE_OK && iOrigin!=0 ){
rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
}
return rc;
}
/*
** Insert a record into the %_docsize table. Specifically, do:
@@ -656,13 +465,6 @@ static int fts5StorageInsertDocsize(
rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pReplace, 1, iRowid);
if( p->pConfig->bContentlessDelete ){
i64 iOrigin = 0;
rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
sqlite3_bind_int64(pReplace, 3, iOrigin);
}
}
if( rc==SQLITE_OK ){
sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
@@ -720,12 +522,7 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(
Fts5Storage *p, /* Storage object */
i64 iDel, /* Rowid to delete from table */
sqlite3_value **apVal, /* Optional - values to remove from index */
int bSaveRow /* If true, set pSavedRow for deleted row */
){
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
Fts5Config *pConfig = p->pConfig;
int rc;
sqlite3_stmt *pDel = 0;
@@ -735,15 +532,7 @@ int sqlite3Fts5StorageDelete(
/* Delete the index records */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
}
if( rc==SQLITE_OK ){
if( p->pConfig->bContentlessDelete ){
rc = fts5StorageContentlessDelete(p, iDel);
}else{
rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
}
rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
}
/* Delete the %_docsize record */
@@ -820,7 +609,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg);
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
@@ -831,36 +620,14 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
int nLoc = 0; /* Size of pLoc in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
sqlite3Fts5ClearLocale(pConfig);
}
const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -942,49 +709,9 @@ int sqlite3Fts5StorageContentInsert(
}else{
sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
int i; /* Counter variable */
int nIndexed = 0; /* Number indexed columns seen */
rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
if( pInsert ) sqlite3_clear_bindings(pInsert);
/* Bind the rowid value */
sqlite3_bind_value(pInsert, 1, apVal[1]);
/* Loop through values for user-defined columns. i=2 is the leftmost
** user-defined column. As is column 1 of pSavedRow. */
for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
int bUnindexed = pConfig->abUnindexed[i-2];
sqlite3_value *pVal = apVal[i];
nIndexed += !bUnindexed;
if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
/* This is an UPDATE statement, and column (i-2) was not modified.
** Retrieve the value from Fts5Storage.pSavedRow instead. */
pVal = sqlite3_column_value(p->pSavedRow, i-1);
if( pConfig->bLocale && bUnindexed==0 ){
sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed,
sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
);
}
}else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
const char *pText = 0;
const char *pLoc = 0;
int nText = 0;
int nLoc = 0;
assert( pConfig->bLocale );
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
if( bUnindexed==0 ){
int iLoc = pConfig->nCol + 1 + nIndexed;
sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
}
}
continue;
}
rc = sqlite3_bind_value(pInsert, i, pVal);
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
@@ -1019,38 +746,14 @@ int sqlite3Fts5StorageIndexInsert(
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
int nLoc = 0; /* Size of pText in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = apVal[ctx.iCol+2];
if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
}
}else{
pVal = apVal[ctx.iCol+2];
}
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
fts5StorageInsertCallback
);
sqlite3Fts5ClearLocale(pConfig);
}
const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -1214,61 +917,29 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i]==0 ){
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(
pVal, &pText, &nText, &pLoc, &nLoc
);
}else{
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = i + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
ctx.iCol = i;
ctx.szCol = 0;
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
);
sqlite3Fts5ClearLocale(pConfig);
}
/* If this is not a columnsize=0 database, check that the number
** of tokens in the value matches the aColSize[] value read from
** the %_docsize table. */
if( rc==SQLITE_OK
&& pConfig->bColumnsize
&& ctx.szCol!=aColSize[i]
){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
int nText = sqlite3_column_bytes(pScan, i+1);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
);
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);
@@ -1387,9 +1058,8 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
assert( p->pConfig->bColumnsize );
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( pLookup ){
if( rc==SQLITE_OK ){
int bCorrupt = 1;
assert( rc==SQLITE_OK );
sqlite3_bind_int64(pLookup, 1, iRowid);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
@@ -1402,8 +1072,6 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
if( bCorrupt && rc==SQLITE_OK ){
rc = FTS5_CORRUPT;
}
}else{
assert( rc!=SQLITE_OK );
}
return rc;
@@ -1449,9 +1117,7 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){
i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
if( p->bTotalsValid ){
rc = fts5StorageSaveTotals(p);
if( rc==SQLITE_OK ){
p->bTotalsValid = 0;
}
p->bTotalsValid = 0;
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexSync(p->pIndex);
+53 -565
View File
@@ -14,7 +14,14 @@
#ifdef SQLITE_TEST
#include "tclsqlite.h"
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#ifdef SQLITE_ENABLE_FTS5
@@ -96,14 +103,14 @@ static int SQLITE_TCLAPI f5tDbAndApi(
rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
sqlite3_step(pStmt);
if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
@@ -237,10 +244,6 @@ static int SQLITE_TCLAPI xF5tApi(
{ "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */
{ "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
{ "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
{ "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */
{ "xInstToken", 2, "IDX ITERM" }, /* 19 */
{ "xColumnLocale", 1, "COL" }, /* 20 */
{ 0, 0, 0}
};
@@ -291,12 +294,12 @@ static int SQLITE_TCLAPI xF5tApi(
break;
}
CASE(3, "xTokenize") {
Tcl_Size nText;
int nText;
char *zText = Tcl_GetStringFromObj(objv[2], &nText);
F5tFunction ctx;
ctx.interp = interp;
ctx.pScript = objv[3];
rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
if( rc==SQLITE_OK ){
Tcl_ResetResult(interp);
}
@@ -392,7 +395,7 @@ static int SQLITE_TCLAPI xF5tApi(
CASE(12, "xSetAuxdata") {
F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));
if( pData==0 ){
Tcl_AppendResult(interp, "out of memory", (char*)0);
Tcl_AppendResult(interp, "out of memory", 0);
return TCL_ERROR;
}
pData->pObj = objv[2];
@@ -452,7 +455,7 @@ static int SQLITE_TCLAPI xF5tApi(
rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);
Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
return TCL_ERROR;
}
for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
@@ -497,52 +500,6 @@ static int SQLITE_TCLAPI xF5tApi(
break;
}
CASE(18, "xQueryToken") {
const char *pTerm = 0;
int nTerm = 0;
int iPhrase = 0;
int iTerm = 0;
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
if( rc==SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
}
break;
}
CASE(19, "xInstToken") {
const char *pTerm = 0;
int nTerm = 0;
int iIdx = 0;
int iTerm = 0;
if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
if( rc==SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
}
break;
}
CASE(20, "xColumnLocale") {
const char *z = 0;
int n = 0;
int iCol;
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
return TCL_ERROR;
}
rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
if( rc==SQLITE_OK && z ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
}
break;
}
default:
assert( 0 );
break;
@@ -613,16 +570,15 @@ static void xF5tFunction(
sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
}else{
Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
int n;
const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
char c = zType[0];
if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
/* Only return a BLOB type if the Tcl variable is a bytearray and
** has no string representation. */
Tcl_Size nn;
unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
}else if( c=='b' && strcmp(zType,"boolean")==0 ){
int n;
Tcl_GetIntFromObj(0, pVar, &n);
sqlite3_result_int(pCtx, n);
}else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -635,9 +591,8 @@ static void xF5tFunction(
Tcl_GetWideIntFromObj(0, pVar, &v);
sqlite3_result_int64(pCtx, v);
}else{
Tcl_Size nn;
unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
}
}
}
@@ -683,7 +638,7 @@ static int SQLITE_TCLAPI f5tCreateFunction(
pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
@@ -730,7 +685,7 @@ static int SQLITE_TCLAPI f5tTokenize(
Tcl_Obj *CONST objv[]
){
char *zText;
Tcl_Size nText;
int nText;
sqlite3 *db = 0;
fts5_api *pApi = 0;
Fts5Tokenizer *pTok = 0;
@@ -739,7 +694,7 @@ static int SQLITE_TCLAPI f5tTokenize(
void *pUserdata;
int rc;
Tcl_Size nArg;
int nArg;
const char **azArg;
F5tTokenizeCtx ctx;
@@ -750,7 +705,7 @@ static int SQLITE_TCLAPI f5tTokenize(
if( objc==5 ){
char *zOpt = Tcl_GetString(objv[1]);
if( strcmp("-subst", zOpt) ){
Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);
Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0);
return TCL_ERROR;
}
}
@@ -759,7 +714,7 @@ static int SQLITE_TCLAPI f5tTokenize(
return TCL_ERROR;
}
if( nArg==0 ){
Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);
Tcl_AppendResult(interp, "no such tokenizer: ", 0);
Tcl_Free((void*)azArg);
return TCL_ERROR;
}
@@ -767,13 +722,13 @@ static int SQLITE_TCLAPI f5tTokenize(
rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);
Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0);
return TCL_ERROR;
}
rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
return TCL_ERROR;
}
@@ -783,11 +738,11 @@ static int SQLITE_TCLAPI f5tTokenize(
ctx.pRet = pRet;
ctx.zInput = zText;
rc = tokenizer.xTokenize(
pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2
pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
);
tokenizer.xDelete(pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
Tcl_DecrRefCount(pRet);
return TCL_ERROR;
}
@@ -811,32 +766,18 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance;
struct F5tTokenizerContext {
void *pCtx;
int (*xToken)(void*, int, const char*, int, int, int);
F5tTokenizerInstance *pInst;
};
struct F5tTokenizerModule {
Tcl_Interp *interp;
Tcl_Obj *pScript;
void *pParentCtx;
fts5_tokenizer_v2 parent_v2;
fts5_tokenizer parent;
F5tTokenizerContext *pContext;
};
/*
** zLocale:
** Within a call to xTokenize_v2(), pLocale/nLocale store the locale
** passed to the call by fts5. This can be retrieved by a Tcl tokenize
** script using [sqlite3_fts5_locale].
*/
struct F5tTokenizerInstance {
Tcl_Interp *interp;
Tcl_Obj *pScript;
F5tTokenizerModule *pModule;
Fts5Tokenizer *pParent;
F5tTokenizerContext *pContext;
const char *pLocale;
int nLocale;
};
static int f5tTokenizerCreate(
@@ -845,20 +786,11 @@ static int f5tTokenizerCreate(
int nArg,
Fts5Tokenizer **ppOut
){
Fts5Tokenizer *pParent = 0;
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
Tcl_Obj *pEval;
int rc = TCL_OK;
int i;
assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
if( pMod->parent_v2.xCreate ){
rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
}
if( pMod->parent.xCreate ){
rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
}
pEval = Tcl_DuplicateObj(pMod->pScript);
Tcl_IncrRefCount(pEval);
for(i=0; rc==TCL_OK && i<nArg; i++){
@@ -878,8 +810,6 @@ static int f5tTokenizerCreate(
pInst->interp = pMod->interp;
pInst->pScript = Tcl_GetObjResult(pMod->interp);
pInst->pContext = pMod->pContext;
pInst->pParent = pParent;
pInst->pModule = pMod;
Tcl_IncrRefCount(pInst->pScript);
*ppOut = (Fts5Tokenizer*)pInst;
}
@@ -890,21 +820,11 @@ static int f5tTokenizerCreate(
static void f5tTokenizerDelete(Fts5Tokenizer *p){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
if( pInst ){
if( pInst->pParent ){
if( pInst->pModule->parent_v2.xDelete ){
pInst->pModule->parent_v2.xDelete(pInst->pParent);
}else{
pInst->pModule->parent.xDelete(pInst->pParent);
}
}
Tcl_DecrRefCount(pInst->pScript);
ckfree((char *)pInst);
}
Tcl_DecrRefCount(pInst->pScript);
ckfree((char *)pInst);
}
static int f5tTokenizerReallyTokenize(
static int f5tTokenizerTokenize(
Fts5Tokenizer *p,
void *pCtx,
int flags,
@@ -912,7 +832,6 @@ static int f5tTokenizerReallyTokenize(
int (*xToken)(void*, int, const char*, int, int, int)
){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
F5tTokenizerInstance *pOldInst = 0;
void *pOldCtx;
int (*xOldToken)(void*, int, const char*, int, int, int);
Tcl_Obj *pEval;
@@ -921,11 +840,9 @@ static int f5tTokenizerReallyTokenize(
pOldCtx = pInst->pContext->pCtx;
xOldToken = pInst->pContext->xToken;
pOldInst = pInst->pContext->pInst;
pInst->pContext->pCtx = pCtx;
pInst->pContext->xToken = xToken;
pInst->pContext->pInst = pInst;
assert(
flags==FTS5_TOKENIZE_DOCUMENT
@@ -961,105 +878,9 @@ static int f5tTokenizerReallyTokenize(
pInst->pContext->pCtx = pOldCtx;
pInst->pContext->xToken = xOldToken;
pInst->pContext->pInst = pOldInst;
return rc;
}
typedef struct CallbackCtx CallbackCtx;
struct CallbackCtx {
Fts5Tokenizer *p;
void *pCtx;
int flags;
int (*xToken)(void*, int, const char*, int, int, int);
};
static int f5tTokenizeCallback(
void *pCtx,
int tflags,
const char *z, int n,
int iStart, int iEnd
){
CallbackCtx *p = (CallbackCtx*)pCtx;
return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
}
static int f5tTokenizerTokenize_v2(
Fts5Tokenizer *p,
void *pCtx,
int flags,
const char *pText, int nText,
const char *pLoc, int nLoc,
int (*xToken)(void*, int, const char*, int, int, int)
){
int rc = SQLITE_OK;
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
pInst->pLocale = pLoc;
pInst->nLocale = nLoc;
if( pInst->pParent ){
CallbackCtx ctx;
ctx.p = p;
ctx.pCtx = pCtx;
ctx.flags = flags;
ctx.xToken = xToken;
if( pInst->pModule->parent_v2.xTokenize ){
rc = pInst->pModule->parent_v2.xTokenize(
pInst->pParent, (void*)&ctx, flags, pText, nText,
pLoc, nLoc, f5tTokenizeCallback
);
}else{
rc = pInst->pModule->parent.xTokenize(
pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
);
}
}else{
rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
}
pInst->pLocale = 0;
pInst->nLocale = 0;
return rc;
}
static int f5tTokenizerTokenize(
Fts5Tokenizer *p,
void *pCtx,
int flags,
const char *pText, int nText,
int (*xToken)(void*, int, const char*, int, int, int)
){
return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
}
/*
** sqlite3_fts5_locale
*/
static int SQLITE_TCLAPI f5tTokenizerLocale(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
if( objc!=1 ){
Tcl_WrongNumArgs(interp, 1, objv, "");
return TCL_ERROR;
}
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0
);
return TCL_ERROR;
}
Tcl_SetObjResult(interp,
Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
);
return TCL_OK;
}
/*
** sqlite3_fts5_token ?-colocated? TEXT START END
*/
@@ -1072,13 +893,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
int iStart;
int iEnd;
Tcl_Size nToken;
int nToken;
int tflags = 0;
char *zToken;
int rc;
if( objc==5 ){
Tcl_Size nArg;
int nArg;
char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
tflags |= FTS5_TOKEN_COLOCATED;
@@ -1098,12 +919,12 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_token may only be used by tokenizer callback", (char*)0
"sqlite3_fts5_token may only be used by tokenizer callback", 0
);
return TCL_ERROR;
}
rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
@@ -1145,112 +966,32 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
fts5_api *pApi;
char *zName;
Tcl_Obj *pScript;
fts5_tokenizer t;
F5tTokenizerModule *pMod;
int rc = SQLITE_OK;
int bV2 = 0; /* True to use _v2 API */
int iVersion = 2; /* Value for _v2.iVersion */
const char *zParent = 0; /* Name of parent tokenizer, if any */
int ii = 0;
int rc;
if( objc<4 ){
Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
return TCL_ERROR;
}
/* Parse any options. Set stack variables bV2 and zParent. */
for(ii=1; ii<objc-3; ii++){
int iOpt = 0;
const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
return TCL_ERROR;
}
switch( iOpt ){
case 0: /* -v2 */ {
bV2 = 1;
break;
}
case 1: /* -parent */ {
ii++;
if( ii==objc-3 ){
Tcl_AppendResult(
interp, "option requires an argument: -parent", (char*)0
);
return TCL_ERROR;
}
zParent = Tcl_GetString(objv[ii]);
break;
}
case 2: /* -version */ {
ii++;
if( ii==objc-3 ){
Tcl_AppendResult(
interp, "option requires an argument: -version", (char*)0
);
return TCL_ERROR;
}
if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
return TCL_ERROR;
}
break;
}
default:
assert( 0 );
break;
}
}
if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
return TCL_ERROR;
}
zName = Tcl_GetString(objv[objc-2]);
pScript = objv[objc-1];
zName = Tcl_GetString(objv[2]);
pScript = objv[3];
t.xCreate = f5tTokenizerCreate;
t.xTokenize = f5tTokenizerTokenize;
t.xDelete = f5tTokenizerDelete;
pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
memset(pMod, 0, sizeof(F5tTokenizerModule));
pMod->interp = interp;
pMod->pScript = pScript;
Tcl_IncrRefCount(pScript);
pMod->pContext = pContext;
if( zParent ){
if( bV2 ){
fts5_tokenizer_v2 *pParent = 0;
rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
if( rc==SQLITE_OK ){
memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
pMod->parent_v2.xDelete(0);
}
}else{
rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
if( rc==SQLITE_OK ){
pMod->parent.xDelete(0);
}
}
}
if( rc==SQLITE_OK ){
void *pModCtx = (void*)pMod;
if( bV2==0 ){
fts5_tokenizer t;
t.xCreate = f5tTokenizerCreate;
t.xTokenize = f5tTokenizerTokenize;
t.xDelete = f5tTokenizerDelete;
rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
}else{
fts5_tokenizer_v2 t2;
memset(&t2, 0, sizeof(t2));
t2.iVersion = iVersion;
t2.xCreate = f5tTokenizerCreate;
t2.xTokenize = f5tTokenizerTokenize_v2;
t2.xDelete = f5tTokenizerDelete;
rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
}
}
Tcl_IncrRefCount(pScript);
rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, (
bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
: "error in fts5_api.xCreateTokenizer()"
), (char*)0);
Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
return TCL_ERROR;
}
@@ -1307,7 +1048,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
Tcl_Obj *CONST objv[]
){
char *z;
Tcl_Size n;
int n;
unsigned int iVal;
int nSlot;
@@ -1320,7 +1061,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
}
z = Tcl_GetStringFromObj(objv[2], &n);
iVal = f5t_fts5HashKey(nSlot, z, (int)n);
iVal = f5t_fts5HashKey(nSlot, z, n);
Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
return TCL_OK;
}
@@ -1376,256 +1117,6 @@ static int SQLITE_TCLAPI f5tRegisterTok(
return TCL_OK;
}
typedef struct OriginTextCtx OriginTextCtx;
struct OriginTextCtx {
sqlite3 *db;
fts5_api *pApi;
};
typedef struct OriginTextTokenizer OriginTextTokenizer;
struct OriginTextTokenizer {
Fts5Tokenizer *pTok; /* Underlying tokenizer object */
fts5_tokenizer tokapi; /* API implementation for pTok */
};
/*
** Delete the OriginTextCtx object indicated by the only argument.
*/
static void f5tOrigintextTokenizerDelete(void *pCtx){
OriginTextCtx *p = (OriginTextCtx*)pCtx;
ckfree((char*)p);
}
static int f5tOrigintextCreate(
void *pCtx,
const char **azArg,
int nArg,
Fts5Tokenizer **ppOut
){
OriginTextCtx *p = (OriginTextCtx*)pCtx;
OriginTextTokenizer *pTok = 0;
void *pTokCtx = 0;
int rc = SQLITE_OK;
pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
if( pTok==0 ){
rc = SQLITE_NOMEM;
}else if( nArg<1 ){
rc = SQLITE_ERROR;
}else{
/* Locate the underlying tokenizer */
rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
}
/* Create the new tokenizer instance */
if( rc==SQLITE_OK ){
rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
}
if( rc!=SQLITE_OK ){
sqlite3_free(pTok);
pTok = 0;
}
*ppOut = (Fts5Tokenizer*)pTok;
return rc;
}
static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
if( p->pTok ){
p->tokapi.xDelete(p->pTok);
}
sqlite3_free(p);
}
typedef struct OriginTextCb OriginTextCb;
struct OriginTextCb {
void *pCtx;
const char *pText;
int nText;
int (*xToken)(void *, int, const char *, int, int, int);
char *aBuf; /* Buffer to use */
int nBuf; /* Allocated size of aBuf[] */
};
static int xOriginToken(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input */
){
OriginTextCb *p = (OriginTextCb*)pCtx;
int ret = 0;
if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
/* Token exactly matches document text. Pass it through as is. */
ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
}else{
int nReq = nToken + 1 + (iEnd-iStart);
if( nReq>p->nBuf ){
sqlite3_free(p->aBuf);
p->aBuf = sqlite3_malloc(nReq*2);
if( p->aBuf==0 ) return SQLITE_NOMEM;
p->nBuf = nReq*2;
}
memcpy(p->aBuf, pToken, nToken);
p->aBuf[nToken] = '\0';
memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
}
return ret;
}
static int f5tOrigintextTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
int (*xToken)(void *, int, const char *, int, int, int)
){
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
OriginTextCb cb;
int ret;
memset(&cb, 0, sizeof(cb));
cb.pCtx = pCtx;
cb.pText = pText;
cb.nText = nText;
cb.xToken = xToken;
ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
sqlite3_free(cb.aBuf);
return ret;
}
/*
** sqlite3_fts5_register_origintext DB
**
** Description...
*/
static int SQLITE_TCLAPI f5tRegisterOriginText(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3 *db = 0;
fts5_api *pApi = 0;
int rc;
fts5_tokenizer tok = {0, 0, 0};
OriginTextCtx *pCtx = 0;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB");
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
pCtx->db = db;
pCtx->pApi = pApi;
tok.xCreate = f5tOrigintextCreate;
tok.xDelete = f5tOrigintextDelete;
tok.xTokenize = f5tOrigintextTokenize;
rc = pApi->xCreateTokenizer(
pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
);
Tcl_ResetResult(interp);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
return TCL_ERROR;
}
return TCL_OK;
}
/*
** This function is used to DROP an fts5 table. It works even if the data
** structures fts5 stores within the database are corrupt, which sometimes
** prevents a straight "DROP TABLE" command from succeeding.
**
** The first parameter is the database handle to use for the DROP TABLE
** operation. The second is the name of the database to drop the fts5 table
** from (i.e. "main", "temp" or the name of an attached database). The
** third parameter is the name of the fts5 table to drop.
**
** SQLITE_OK is returned if the table is successfully dropped. Or, if an
** error occurs, an SQLite error code.
*/
static int sqlite3_fts5_drop_corrupt_table(
sqlite3 *db, /* Database handle */
const char *zDb, /* Database name ("main", "temp" etc.) */
const char *zTab /* Name of fts5 table to drop */
){
int rc = SQLITE_OK;
int bDef = 0;
rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef);
if( rc==SQLITE_OK ){
char *zScript = sqlite3_mprintf(
"DELETE FROM %Q.'%q_data';"
"DELETE FROM %Q.'%q_config';"
"INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');"
"INSERT INTO %Q.'%q_config' VALUES('version', 4);"
"DROP TABLE %Q.'%q';",
zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
);
if( zScript==0 ){
rc = SQLITE_NOMEM;
}else{
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
rc = sqlite3_exec(db, zScript, 0, 0, 0);
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
sqlite3_free(zScript);
}
}
return rc;
}
/*
** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE
**
** Description...
*/
static int SQLITE_TCLAPI f5tDropCorruptTable(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3 *db = 0;
const char *zDb = 0;
const char *zTab = 0;
int rc = SQLITE_OK;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE");
return TCL_ERROR;
}
if( f5tDbPointer(interp, objv[1], &db) ){
return TCL_ERROR;
}
zDb = Tcl_GetString(objv[2]);
zTab = Tcl_GetString(objv[3]);
rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
return TCL_ERROR;
}
return TCL_OK;
}
/*
** Entry point.
*/
@@ -1637,15 +1128,12 @@ int Fts5tcl_Init(Tcl_Interp *interp){
} aCmd[] = {
{ "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
{ "sqlite3_fts5_token", f5tTokenizerReturn, 1 },
{ "sqlite3_fts5_locale", f5tTokenizerLocale, 1 },
{ "sqlite3_fts5_tokenize", f5tTokenize, 0 },
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 },
{ "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 },
{ "sqlite3_fts5_token_hash", f5tTokenHash, 0 },
{ "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
{ "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
{ "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
};
int i;
F5tTokenizerContext *pContext;
+1 -2
View File
@@ -472,8 +472,7 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0 /* xIntegrity */
0 /* xShadowName */
};
int rc; /* Return code */
+52 -124
View File
@@ -229,12 +229,6 @@ static const unsigned char sqlite3Utf8Trans1[] = {
#endif /* ifndef SQLITE_AMALGAMATION */
#define FTS5_SKIP_UTF8(zIn) { \
if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
} \
}
typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
unsigned char aTokenChar[128]; /* ASCII range token characters */
@@ -386,6 +380,7 @@ static int fts5UnicodeCreate(
zCat = azArg[i+1];
}
}
if( rc==SQLITE_OK ){
rc = unicodeSetCategories(p, zCat);
}
@@ -415,6 +410,7 @@ static int fts5UnicodeCreate(
rc = SQLITE_ERROR;
}
}
}else{
rc = SQLITE_NOMEM;
}
@@ -553,7 +549,7 @@ static int fts5UnicodeTokenize(
typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
fts5_tokenizer tokenizer; /* Parent tokenizer module */
Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};
@@ -565,7 +561,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){
if( pTok ){
PorterTokenizer *p = (PorterTokenizer*)pTok;
if( p->pTokenizer ){
p->tokenizer_v2.xDelete(p->pTokenizer);
p->tokenizer.xDelete(p->pTokenizer);
}
sqlite3_free(p);
}
@@ -584,7 +580,6 @@ static int fts5PorterCreate(
PorterTokenizer *pRet;
void *pUserdata = 0;
const char *zBase = "unicode61";
fts5_tokenizer_v2 *pV2 = 0;
if( nArg>0 ){
zBase = azArg[0];
@@ -593,15 +588,14 @@ static int fts5PorterCreate(
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
int nArg2 = (nArg>0 ? nArg-1 : 0);
const char **az2 = (nArg2 ? &azArg[1] : 0);
memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
const char **azArg2 = (nArg2 ? &azArg[1] : 0);
rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
}
if( rc!=SQLITE_OK ){
@@ -1252,7 +1246,6 @@ static int fts5PorterTokenize(
void *pCtx,
int flags,
const char *pText, int nText,
const char *pLoc, int nLoc,
int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
@@ -1260,8 +1253,8 @@ static int fts5PorterTokenize(
sCtx.xToken = xToken;
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer_v2.xTokenize(
p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
return p->tokenizer.xTokenize(
p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
);
}
@@ -1271,7 +1264,6 @@ static int fts5PorterTokenize(
typedef struct TrigramTokenizer TrigramTokenizer;
struct TrigramTokenizer {
int bFold; /* True to fold to lower-case */
int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
};
/*
@@ -1291,46 +1283,28 @@ static int fts5TriCreate(
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
TrigramTokenizer *pNew = 0;
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
UNUSED_PARAM(pUnused);
if( nArg%2 ){
rc = SQLITE_ERROR;
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
pNew->bFold = 1;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
}else{
rc = SQLITE_ERROR;
}
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
*ppOut = (Fts5Tokenizer*)pNew;
@@ -1350,62 +1324,40 @@ static int fts5TriTokenize(
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
int rc = SQLITE_OK;
char aBuf[32];
char *zOut = aBuf;
int ii;
const unsigned char *zIn = (const unsigned char*)pText;
const unsigned char *zEof = &zIn[nText];
u32 iCode;
int aStart[3]; /* Input offset of each character in aBuf[] */
UNUSED_PARAM(unusedFlags);
/* Populate aBuf[] with the characters for the first trigram. */
for(ii=0; ii<3; ii++){
do {
aStart[ii] = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) return SQLITE_OK;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
}while( iCode==0 );
WRITE_UTF8(zOut, iCode);
}
/* At the start of each iteration of this loop:
**
** aBuf: Contains 3 characters. The 3 characters of the next trigram.
** zOut: Points to the byte following the last character in aBuf.
** aStart[3]: Contains the byte offset in the input text corresponding
** to the start of each of the three characters in the buffer.
*/
assert( zIn<=zEof );
while( 1 ){
int iNext; /* Start of character following current tri */
const char *z1;
char *zOut = aBuf;
int iStart = zIn - (const unsigned char*)pText;
const unsigned char *zNext;
/* Read characters from the input up until the first non-diacritic */
do {
iNext = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
zNext = zIn;
if( zIn<zEof ){
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
}while( iCode==0 );
/* Pass the current trigram back to fts5 */
rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
if( iCode==0 || rc!=SQLITE_OK ) break;
/* Remove the first character from buffer aBuf[]. Append the character
** with codepoint iCode. */
z1 = aBuf;
FTS5_SKIP_UTF8(z1);
memmove(aBuf, z1, zOut - z1);
zOut -= (z1 - aBuf);
WRITE_UTF8(zOut, iCode);
/* Update the aStart[] array */
aStart[0] = aStart[1];
aStart[1] = aStart[2];
aStart[2] = iNext;
}else{
break;
}
if( zIn<zEof ){
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
}else{
break;
}
rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
if( rc!=SQLITE_OK ) break;
zIn = zNext;
}
return rc;
@@ -1428,23 +1380,11 @@ int sqlite3Fts5TokenizerPattern(
){
if( xCreate==fts5TriCreate ){
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
if( p->iFoldParam==0 ){
return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
}
return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
}
return FTS5_PATTERN_NONE;
}
/*
** Return true if the tokenizer described by p->azArg[] is the trigram
** tokenizer. This tokenizer needs to be loaded before xBestIndex is
** called for the first time in order to correctly handle LIKE/GLOB.
*/
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
}
/*
** Register all built-in tokenizers with FTS5.
*/
@@ -1455,6 +1395,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
} aBuiltin[] = {
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
{ "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
{ "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
};
@@ -1469,19 +1410,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
0
);
}
if( rc==SQLITE_OK ){
fts5_tokenizer_v2 sPorter = {
2,
fts5PorterCreate,
fts5PorterDelete,
fts5PorterTokenize
};
rc = pApi->xCreateTokenizer_v2(pApi,
"porter",
(void*)pApi,
&sPorter,
0
);
}
return rc;
}
-3
View File
@@ -364,9 +364,6 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
default: return 1; }
break;
default:
return 1;
}
return 0;
}
+11 -28
View File
@@ -64,7 +64,6 @@ struct Fts5VocabCursor {
int nLeTerm; /* Size of zLeTerm in bytes */
char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
int colUsed; /* Copy of sqlite3_index_info.colUsed */
/* These are used by 'col' tables only */
int iCol;
@@ -91,11 +90,9 @@ struct Fts5VocabCursor {
/*
** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
*/
#define FTS5_VOCAB_TERM_EQ 0x0100
#define FTS5_VOCAB_TERM_GE 0x0200
#define FTS5_VOCAB_TERM_LE 0x0400
#define FTS5_VOCAB_COLUSED_MASK 0xFF
#define FTS5_VOCAB_TERM_EQ 0x01
#define FTS5_VOCAB_TERM_GE 0x02
#define FTS5_VOCAB_TERM_LE 0x04
/*
@@ -272,13 +269,11 @@ static int fts5VocabBestIndexMethod(
int iTermEq = -1;
int iTermGe = -1;
int iTermLe = -1;
int idxNum = (int)pInfo->colUsed;
int idxNum = 0;
int nArg = 0;
UNUSED_PARAM(pUnused);
assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
for(i=0; i<pInfo->nConstraint; i++){
struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
if( p->usable==0 ) continue;
@@ -370,7 +365,7 @@ static int fts5VocabOpenMethod(
if( rc==SQLITE_OK ){
pVTab->zErrMsg = sqlite3_mprintf(
"no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
);
);
rc = SQLITE_ERROR;
}
}else{
@@ -379,7 +374,7 @@ static int fts5VocabOpenMethod(
}
if( rc==SQLITE_OK ){
i64 nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor);
int nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor);
pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
}
@@ -530,19 +525,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
switch( pTab->eType ){
case FTS5_VOCAB_ROW:
/* Do not bother counting the number of instances if the "cnt"
** column is not being read (according to colUsed). */
if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
while( iPos<nPos ){
u32 ii;
fts5FastGetVarint32(pPos, iPos, ii);
if( ii==1 ){
/* New column in the position list */
fts5FastGetVarint32(pPos, iPos, ii);
}else{
/* An instance - increment pCsr->aCnt[] */
pCsr->aCnt[0]++;
}
if( eDetail==FTS5_DETAIL_FULL ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
}
}
pCsr->aDoc[0]++;
@@ -640,12 +625,11 @@ static int fts5VocabFilterMethod(
if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);
if( pEq ){
zTerm = (const char *)sqlite3_value_text(pEq);
nTerm = sqlite3_value_bytes(pEq);
f = FTS5INDEX_QUERY_NOTOKENDATA;
f = 0;
}else{
if( pGe ){
zTerm = (const char *)sqlite3_value_text(pGe);
@@ -799,8 +783,7 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
/* xSavepoint */ 0,
/* xRelease */ 0,
/* xRollbackTo */ 0,
/* xShadowName */ 0,
/* xIntegrity */ 0
/* xShadowName */ 0
};
void *p = (void*)pGlobal;
-61
View File
@@ -51,10 +51,6 @@ proc fts5_test_poslist2 {cmd} {
sort_poslist $res
}
proc fts5_test_insttoken {cmd iInst iToken} {
$cmd xInstToken $iInst $iToken
}
proc fts5_test_collist {cmd} {
set res [list]
@@ -65,12 +61,6 @@ proc fts5_test_collist {cmd} {
set res
}
proc fts5_collist {cmd iPhrase} {
set res [list]
$cmd xPhraseColumnForeach $iPhrase c { lappend res $c }
set res
}
proc fts5_test_columnsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -79,13 +69,6 @@ proc fts5_test_columnsize {cmd} {
set res
}
proc fts5_columntext {cmd iCol} {
$cmd xColumnText $iCol
}
proc fts5_columnlocale {cmd iCol} {
$cmd xColumnLocale $iCol
}
proc fts5_test_columntext {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -94,14 +77,6 @@ proc fts5_test_columntext {cmd} {
set res
}
proc fts5_test_columnlocale {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnLocale $i]
}
set res
}
proc fts5_test_columntotalsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -129,10 +104,6 @@ proc fts5_test_rowcount {cmd} {
$cmd xRowCount
}
proc fts5_test_rowid {cmd} {
$cmd xRowid
}
proc test_queryphrase_cb {cnt cmd} {
upvar $cnt L
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -154,13 +125,6 @@ proc fts5_test_queryphrase {cmd} {
set res
}
proc fts5_queryphrase {cmd iPhrase} {
set cnt [list]
for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
$cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt]
set cnt
}
proc fts5_test_phrasecount {cmd} {
$cmd xPhraseCount
}
@@ -180,23 +144,16 @@ proc fts5_aux_test_functions {db} {
foreach f {
fts5_test_columnsize
fts5_test_columntext
fts5_test_columnlocale
fts5_test_columntotalsize
fts5_test_poslist
fts5_test_poslist2
fts5_test_collist
fts5_test_insttoken
fts5_test_tokenize
fts5_test_rowcount
fts5_test_rowid
fts5_test_all
fts5_test_queryphrase
fts5_test_phrasecount
fts5_columntext
fts5_columnlocale
fts5_queryphrase
fts5_collist
} {
sqlite3_fts5_create_function $db $f $f
}
@@ -481,20 +438,6 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
proc foreach_tokenizer_mode {prefix script} {
set saved $::testprefix
foreach {d mapping} {
"" {}
"-origintext" {, tokenize="origintext unicode61", tokendata=1}
} {
set s [string map [list %TOKENIZER% $mapping] $script]
set ::testprefix "$prefix$d"
reset_db
sqlite3_fts5_register_origintext db
uplevel $s
}
set ::testprefix $saved
}
#-------------------------------------------------------------------------
# Convert a poslist of the type returned by fts5_test_poslist() to a
@@ -651,10 +594,6 @@ proc nearset_rc {aCol args} {
list
}
proc dump {tname} {
execsql_pp "SELECT * FROM ${tname}_idx"
execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
}
#-------------------------------------------------------------------------
# Code for a simple Tcl tokenizer that supports synonyms at query time.
+28 -51
View File
@@ -22,7 +22,6 @@ ifcapable !fts5 {
}
foreach_detail_mode $::testprefix {
foreach_tokenizer_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -45,7 +44,7 @@ do_execsql_test 1.1 {
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -66,17 +65,14 @@ foreach w {a b c d e f} {
do_execsql_test 2.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
PRAGMA integrity_check;
PRAGMA integrity_check(t1);
} {ok ok}
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
@@ -92,16 +88,14 @@ foreach {i x y} {
} {
do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@@ -124,9 +118,8 @@ foreach {i x y} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@@ -142,16 +135,15 @@ foreach {i x y} {
10 {ddd abcde dddd dd c} {dddd c c d abcde}
} {
do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok
do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
@@ -186,7 +178,6 @@ do_execsql_test 6.6 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
expr srand(0)
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -228,7 +219,6 @@ for {set i 1} {$i <= 10} {incr i} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -243,7 +233,6 @@ do_execsql_test 8.1 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
expr srand(0)
@@ -288,9 +277,8 @@ for {set i 1} {$i <= 10} {incr i} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
set d10 {
1 {g f d b f} {h h e i a}
@@ -323,19 +311,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
} {1 {reserved fts5 column name: rowid}}
#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
} {}
do_catchsql_test 12.2 {
@@ -350,9 +338,8 @@ do_test 12.3 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
} {}
@@ -375,9 +362,8 @@ do_execsql_test 13.6 {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -440,7 +426,7 @@ do_execsql_test 16.1 {
proc funk {} {
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
set fd [db incrblob main n1_data block 10]
fconfigure $fd -translation binary
fconfigure $fd -encoding binary -translation binary
# puts -nonewline $fd "\x44\x45"
close $fd
}
@@ -453,16 +439,15 @@ db func funk funk
# statement no longer fails.
#
do_catchsql_test 16.2 {
SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {0 {{} -1e-06 {}}}
# {1 {SQL logic error}}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
INSERT INTO b2 VALUES('a');
INSERT INTO b2 VALUES('b');
INSERT INTO b2 VALUES('c');
@@ -478,9 +463,8 @@ do_test 17.2 {
if {[string match n* %DETAIL%]==0} {
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.3 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
@@ -489,9 +473,8 @@ if {[string match n* %DETAIL%]==0} {
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
INSERT INTO uio VALUES(NULL);
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
@@ -538,8 +521,8 @@ do_execsql_test 17.9 {
#--------------------------------------------------------------------
#
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
INSERT INTO t1 VALUES('abc*', NULL);
INSERT INTO t2 VALUES(1, 'abcdefg');
}
@@ -554,9 +537,8 @@ do_execsql_test 18.3 {
# fts5 table in the temp schema.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('w x 1');
SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -566,9 +548,8 @@ do_execsql_test 19.0 {
# Test that 6 and 7 byte varints can be read.
#
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
}
set ::ids [list \
0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -586,7 +567,7 @@ do_test 20.1 {
#
do_execsql_test 21.0 {
CREATE TEMP TABLE t8(a, b);
CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
}
do_execsql_test 21.1 {
@@ -597,7 +578,7 @@ do_execsql_test 21.1 {
}
do_execsql_test 22.0 {
CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
BEGIN;
INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -612,7 +593,7 @@ do_execsql_test 22.1 {
#-------------------------------------------------------------------------
do_execsql_test 23.0 {
CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
CREATE TABLE t11(x);
}
do_execsql_test 23.1 {
@@ -624,7 +605,7 @@ do_execsql_test 23.2 {
#-------------------------------------------------------------------------
do_execsql_test 24.0 {
CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
INSERT INTO t12 VALUES('aaaa');
}
do_execsql_test 24.1 {
@@ -634,9 +615,6 @@ do_execsql_test 24.1 {
INSERT INTO t12 VALUES('aaaa');
END;
}
execsql_pp {
SELECT rowid, hex(block) FROM t12_data
}
do_execsql_test 24.2 {
INSERT INTO t12(t12) VALUES('integrity-check');
}
@@ -646,7 +624,7 @@ do_execsql_test 24.3 {
#-------------------------------------------------------------------------
do_execsql_test 25.0 {
CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
}
do_execsql_test 25.1 {
BEGIN;
@@ -657,7 +635,6 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
}
}
}
expand_all_sql db
+2 -6
View File
@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ab
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -180,11 +180,7 @@ if {[detail_is_full]} {
} {1 2}
}
do_execsql_test 4.5.1 {
SELECT rowid FROM s1 WHERE s1 MATCH 'a AND x'
} {1 2}
do_execsql_test 4.5.2 {
do_execsql_test 4.5 {
SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
} {1 2}
+1 -1
View File
@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ac
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+1 -1
View File
@@ -18,7 +18,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ad
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+1 -1
View File
@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ae
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+2 -38
View File
@@ -18,7 +18,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5af
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -165,7 +165,7 @@ do_execsql_test 4.0 {
}
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE p1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE p1 USING fts5(a, b);
INSERT INTO p1 VALUES(
'x a a a a a a a a a a',
'a a a a a a a a a a a a a a a a a a a x'
@@ -184,43 +184,7 @@ do_execsql_test 5.3 {
do_execsql_test 5.4 {
SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
} {{[x a a a a a...}}
do_execsql_test 5.5 {
SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x OR ""');
} {{[x a a a a a...}}
do_execsql_test 5.6 {
SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x OR ' || x'DB');
} {{[x a a a a a...}}
} ;# foreach_detail_mode
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(colA, colB);
INSERT INTO t1 VALUES('A B C', 'D E F');
}
do_execsql_test 6.1 {
SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B';
} {{A B C} {D E F} ...[B]...}
breakpoint
do_execsql_test 6.2 {
SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B';
} {{A B C} {D E F} {D E...}}
do_execsql_test 6.3 {
SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B';
} {{A B C} {D E F} {D...}}
do_execsql_test 6.1 {
SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A';
} {{A B C} {D E F} [A]...}
breakpoint
do_execsql_test 6.2 {
SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A';
} {{A B C} {D E F} {D E...}}
do_execsql_test 6.3 {
SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A';
} {{A B C} {D E F} {D...}}
finish_test
+1 -1
View File
@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ag
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+1 -13
View File
@@ -11,12 +11,11 @@
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# TESTRUNNER: slow
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ah
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -163,17 +162,6 @@ do_execsql_test 1.8.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
} {10000}
do_execsql_test 1.8.3 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text';
} {4999}
do_execsql_test 1.8.4 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text';
} {0}
do_catchsql_test 1.9 {
SELECT * FROM t1('*xy');
} {1 {unknown special query: xy}}
} ;# foreach_detail_mode
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
+1 -1
View File
@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ai
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+1 -1
View File
@@ -19,7 +19,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aj
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
+1 -27
View File
@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ak
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -154,30 +154,4 @@ do_execsql_test 3.2 {
}
# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b
#
# This is not a test of FTS5, but rather a test of the of what happens to
# prepared statements that encounter SQLITE_SCHEMA while other prepared
# statements are running. The original problem POC used FTS5, and so
# is seems reasonable to put the test here.
#
# The vdbeaux24.test module in TH3 also tests this same behavior but
# without requiring FTS5 or an other extension.
#
reset_db
db null NULL
do_execsql_test 4.0 {
CREATE TABLE t5(a PRIMARY KEY);
INSERT INTO t5 VALUES(0);
CREATE VIRTUAL TABLE t6 USING fts5(0);
DELETE FROM t6;
CREATE TABLE t7(x);
WITH cte(a) AS (
SELECT a FROM t5
WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0)
< (a,0)
)
SELECT max(a) FROM cte;
} NULL
finish_test
+1 -11
View File
@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5al
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -293,16 +293,6 @@ do_catchsql_test 4.4.4 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}
# Check that the second and subsequent rank= constraints are ignored.
#
do_catchsql_test 4.3.3 {
SELECT *, rank FROM t3
WHERE t3 MATCH 'a' AND
rank MATCH 'nosuch()' AND
rank MATCH 'rowidmod(3)'
ORDER BY rank ASC
} {1 {unable to use function MATCH in the requested context}}
} ;# foreach_detail_mode
+1 -1
View File
@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5alter
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return

Some files were not shown because too many files have changed in this diff Show More