paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change things so that locale=1 is required to write fts5_locale() values to an fts5 table, and so that blobs may not be stored in indexed (i.e. not UNINDEXED) columns of these tables.

Browse Source 
FossilOrigin-Name: c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e
This commit is contained in:
dan
2024-08-02 21:06:13 +00:00
parent f971c15b1f
commit e317e7f4b4
8 changed files with 254 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5Int.h
-11
View File
@@ -208,12 +208,6 @@ struct Fts5TokenizerConfig {
**
** bLocale:
** Set to true if locale=1 was specified when the table was created.
**
** eEnc:
** Set to either FTS5_ENCODING_UNKNOWN, ENCODING_UTF8, or ENCODING_UTF16,
** to indicate the encoding used by the database handle. This is initially
** set to UNKNOWN, then to one of the other two values the first time it
** is required.
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
@@ -236,7 +230,6 @@ struct Fts5Config {
char *zContentExprlist;
Fts5TokenizerConfig t;
int bLock; /* True when table is preparing statement */
int eEnc; /* An FTS5_ENCODING_XXX constant */
/* Values loaded from the %_config table */
@@ -278,10 +271,6 @@ struct Fts5Config {
#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
#define FTS5_ENCODING_UNKNOWN 0
#define FTS5_ENCODING_UTF8 1
#define FTS5_ENCODING_UTF16 2
int sqlite3Fts5ConfigParse(
Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
);
ext/fts5/fts5_config.c
-7
View File
@@ -615,13 +615,6 @@ int sqlite3Fts5ConfigParse(
sqlite3_free(zTwo);
}
/* If this is not an FTS5_CONTENT_NORMAL table, set bLocale. There are
** no restrictions on using fts5_locale() with external-content or
** contentless tables. */
if( pRet->eContent!=FTS5_CONTENT_NORMAL ){
pRet->bLocale = 1;
}
/* We only allow contentless_delete=1 if the table is indeed contentless. */
if( rc==SQLITE_OK
&& pRet->bContentlessDelete
ext/fts5/fts5_main.c
+98 -122
View File
@@ -244,6 +244,7 @@ struct Fts5Cursor {
** The subtype values returned by fts5_locale() are tagged with.
*/
#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L')
#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB"
/*
@@ -1268,43 +1269,6 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
fts5SetLocale(pConfig, 0, 0);
}
/*
** This function is used to determine if the database handle uses utf-8 or
** a utf-16 encoding. If it uses utf-8, then output parameter (*pbIs) is set
** to 0. Otherwise, if it uses utf-16, then the output parameter is set
** to 1.
**
** This function returns SQLITE_OK if successful, or an SQLite error code
** otherwise. If an error code is returned, the final value of (*pbIs) is
** undefined.
*/
static int fts5IsUtf16(
Fts5Config *pConfig, /* Configuration object */
int *pbIs /* OUT: True if utf-16, false if utf-8 */
){
if( pConfig->eEnc==FTS5_ENCODING_UNKNOWN ){
sqlite3_stmt *pPragma = 0;
int rc = fts5PrepareStatement(&pPragma, pConfig,
"SELECT (encoding LIKE '%%16%%') FROM pragma_encoding"
);
if( rc==SQLITE_OK ){
int val;
sqlite3_step(pPragma);
val = sqlite3_column_int(pPragma, 0);
rc = sqlite3_finalize(pPragma);
if( rc!=SQLITE_OK ) return rc;
if( val ){
pConfig->eEnc = FTS5_ENCODING_UTF16;
}else{
pConfig->eEnc = FTS5_ENCODING_UTF8;
}
}
}
*pbIs = (pConfig->eEnc==FTS5_ENCODING_UTF16);
return SQLITE_OK;
}
/*
** This function is used to extract utf-8 text from an sqlite3_value. This
** is usually done in order to tokenize it. For example, when:
@@ -1315,25 +1279,14 @@ static int fts5IsUtf16(
**
** and so on.
**
** This function handles 3 cases:
** This function handles 2 cases:
**
** 1) Ordinary values. The text can be extracted from these using
** sqlite3_value_text().
**
** 2) Blobs tagged with sub-type FTS5_LOCALE_SUBTYPE, or those read from
** the content table of an FTS5_CONTENT_NORMAL table with locale=1
** set that do not begin with 0x00.
**
** In these cases the value is a blob formatted by fts5_locale() that
** contains both a locale and a text value. The locale is first, as
** utf-8, followed by a single 0x00 byte, followed by the text value,
** also as utf-8. There is no nul-terminator for the text value.
**
** 3) Blobs read from the content table of an FTS5_CONTENT_NORMAL table
** with locale=1 set that do begin with 0x00. These are used to
** store actual SQLITE_BLOB values written to the fts5 table by the
** user. They begin with 4 0x00 bytes, followed by the blob data as
** specified by the user.
** the content table of a normal content or external-conten table
** with locale=1 set.
**
** If successful, SQLITE_OK is returned and output parameters (*ppText)
** and (*pnText) are set to point to a buffer containing the extracted utf-8
@@ -1342,9 +1295,7 @@ static int fts5IsUtf16(
** from which it is extracted.
**
** Parameter bContent must be true if the value was read from an indexed
** column (i.e. not UNINDEXED) of the on disk content. In this case, if
** the table is FTS5_CONTENT_NORMAL and locale=1 was specified, special blob
** cases (2) and (3) above will apply.
** column (i.e. not UNINDEXED) of the on disk content.
**
** If pbResetTokenizer is not NULL and if case (2) is used, then the
** tokenizer is configured to use the locale. In this case (*pbResetTokenizer)
@@ -1366,57 +1317,50 @@ int sqlite3Fts5ExtractText(
int bDecodeBlob = 0;
assert( pbResetTokenizer==0 || *pbResetTokenizer==0 );
assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE );
assert( bContent==0 || sqlite3_value_subtype(pVal)==0 );
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE
|| (bContent && pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL)
|| (bContent && pConfig->bLocale)
){
bDecodeBlob = 1;
}
}
if( bDecodeBlob ){
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
int nLocale = 0;
if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){
/* Extract the text from a blob stored in a locale=1 table. The
** value consists of 4 0x00 bytes followed by the blob specified
** by the user. The extracted text has to match the text extracted
** when the blob was inserted - by calling sqlite3_value_text() on
** the value without the 4 0x00 byte header.
**
** The tricky bit here is that the exact text extracted from a blob
** depends on the encoding of the database. To avoid reimplementing
** SQLite's blob-to-text conversion code here, we call
** sqlite3_value_text() on the blob with the header, then trim off the
** leading utf-8 characters that the 4 byte header was converted to. In
** practice this is 4 0x00 bytes for a utf-8 database, or 2 0x00 bytes
** for a utf-16 database. */
int bIs16 = 0; /* True for utf-16 database */
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
rc = fts5IsUtf16(pConfig, &bIs16);
if( bIs16 ){
pText += 2;
nText -= 2;
/* Unless this blob was read from the %_content table of an
** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale()
** header. Check for this. If it is not found, return an error. */
if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){
if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
rc = SQLITE_ERROR;
}else{
pText += 4;
nText -= 4;
pBlob += 4;
nBlob -= 4;
}
}else{
}
if( rc==SQLITE_OK ){
int nLocale = 0;
for(nLocale=0; nLocale<nBlob; nLocale++){
if( pBlob[nLocale]==0x00 ) break;
}
if( nLocale==nBlob || nLocale==0 ){
rc = SQLITE_ERROR;
}else{
pText = (const char*)&pBlob[nLocale+1];
nText = nBlob-nLocale-1;
if( nLocale==nBlob || nLocale==0 ) return SQLITE_ERROR;
pText = (const char*)&pBlob[nLocale+1];
nText = nBlob-nLocale-1;
if( pbResetTokenizer ){
rc = fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
*pbResetTokenizer = 1;
if( pbResetTokenizer ){
rc = fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
*pbResetTokenizer = 1;
}
}
}
@@ -2012,6 +1956,25 @@ static int fts5UpdateMethod(
else{
int eType1 = sqlite3_value_numeric_type(apVal[1]);
/* Ensure that no fts5_locale() values are written to locale=0 tables.
** And that no blobs except fts5_locale() blobs are written to indexed
** (i.e. not UNINDEXED) columns of locale=1 tables. */
int ii;
for(ii=0; ii<pConfig->nCol; ii++){
if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){
int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE);
if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0)
|| (pConfig->bLocale==0 && bSub)
){
if( pConfig->bLocale==0 ){
fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
}
rc = SQLITE_MISMATCH;
goto update_out;
}
}
}
if( eType0!=SQLITE_INTEGER ){
/* An INSERT statement. If the conflict-mode is REPLACE, first remove
** the current entry (if any). */
@@ -2074,6 +2037,7 @@ static int fts5UpdateMethod(
}
}
update_out:
pTab->p.pConfig->pzErrmsg = 0;
return rc;
}
@@ -2712,11 +2676,17 @@ static int fts5ApiColumnLocale(
*/
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL
|| sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE
){
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
rc = SQLITE_MISMATCH;
}
pBlob += 4;
nBlob -= 4;
}
if( rc==SQLITE_OK ){
int nLocale = 0;
for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
if( nLocale==nBlob ){
@@ -2967,15 +2937,7 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
**
** 1) Ordinary values. These can be returned using sqlite3_result_value().
**
** 2) Blobs with subtype FTS5_LOCALE_SUBTYPE, or read from an
** FTS5_CONTENT_NORMAL table that do not begin with 0x00. These are
** locale/text pairs. In this case the text is extracted and returned
** via sqlite3_result_text().
**
** 3) Blobs with subtype FTS5_LOCALE_SUBTYPE, or read from an
** FTS5_CONTENT_NORMAL table that begin with 4 0x00 bytes. These are
** blobs with a 4 byte header. In this case the user's blob is extracted
** and returned via sqlite3_result_blob().
** 2) Blobs from fts5_locale().
*/
static void fts5ExtractValueFromColumn(
sqlite3_context *pCtx,
@@ -2983,28 +2945,33 @@ static void fts5ExtractValueFromColumn(
int iCol,
sqlite3_value *pVal
){
assert( pConfig->eContent!=FTS5_CONTENT_NONE );
if( pConfig->bLocale
&& sqlite3_value_type(pVal)==SQLITE_BLOB
&& pConfig->abUnindexed[iCol]==0
){
if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE
|| pConfig->eContent==FTS5_CONTENT_NORMAL
){
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
int ii;
if( nBlob>=4 && memcmp(pBlob, "\0\0\0\0", 4)==0 ){
sqlite3_result_blob(pCtx, &pBlob[4], nBlob-4, SQLITE_TRANSIENT);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
if( nBlob<SZHDR || memcmp(pBlob, FTS5_LOCALE_HEADER, SZHDR) ){
sqlite3_result_error_code(pCtx, SQLITE_MISMATCH);
return;
}else{
int ii;
for(ii=0; ii<nBlob && pBlob[ii]; ii++);
if( ii<nBlob ){
const char *pText = (const char*)&pBlob[ii+1];
sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT);
}
pBlob += 4;
nBlob -= 4;
}
return;
}
for(ii=0; ii<nBlob && pBlob[ii]; ii++);
if( ii<nBlob ){
const char *pText = (const char*)&pBlob[ii+1];
sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT);
}
return;
}
sqlite3_result_value(pCtx, pVal);
@@ -3441,6 +3408,7 @@ static void fts5SourceIdFunc(
** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as
** text, and the value returned is a blob consisting of:
**
** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER).
** * The LOCALE, as utf-8 text, followed by
** * 0x00, followed by
** * The TEXT, as utf-8 text.
@@ -3452,9 +3420,6 @@ static void fts5LocaleFunc(
int nArg, /* Number of args */
sqlite3_value **apArg /* Function arguments */
){
u8 *pBlob = 0;
int nBlob = 0;
const char *zLocale = 0;
int nLocale = 0;
const char *zText = 0;
@@ -3472,16 +3437,27 @@ static void fts5LocaleFunc(
if( zLocale==0 || zLocale[0]=='\0' ){
sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT);
}else{
nBlob = nLocale + 1 + nText;
u8 *pBlob = 0;
u8 *pCsr = 0;
int nBlob = 0;
const int nHdr = 4;
assert( sizeof(FTS5_LOCALE_HEADER)==nHdr+1 );
nBlob = nHdr + nLocale + 1 + nText;
pBlob = (u8*)sqlite3_malloc(nBlob);
if( pBlob==0 ){
sqlite3_result_error_nomem(pCtx);
return;
}
memcpy(pBlob, zLocale, nLocale);
pBlob[nLocale] = 0x00;
if( zText ) memcpy(&pBlob[nLocale+1], zText, nText);
pCsr = pBlob;
memcpy(pCsr, FTS5_LOCALE_HEADER, nHdr);
pCsr += nHdr;
memcpy(pCsr, zLocale, nLocale);
pCsr += nLocale;
(*pCsr++) = 0x00;
if( zText ) memcpy(pCsr, zText, nText);
assert( &pCsr[nText]==&pBlob[nBlob] );
sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free);
sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE);
ext/fts5/fts5_storage.c
+18 -28
View File
@@ -517,7 +517,9 @@ static int fts5StorageDeleteFromIndex(
pVal = apVal[iCol-1];
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
rc = sqlite3Fts5ExtractText(
pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
);
if( rc==SQLITE_OK ){
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
@@ -890,35 +892,23 @@ int sqlite3Fts5StorageContentInsert(
** Retrieve the value from Fts5Storage.pSavedRow instead. */
pVal = sqlite3_column_value(p->pSavedRow, i-1);
}else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
if( pConfig->bLocale==0 ){
sqlite3Fts5ConfigErrmsg(pConfig,
"fts5_locale() may not be used without locale=1"
);
rc = SQLITE_ERROR;
break;
}else if( i>1 && pConfig->abUnindexed[i-2] ){
/* At attempt to insert an fts5_locale() value into an UNINDEXED
** column. Strip the locale away and just bind the text. */
const char *pText = 0;
int nText = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
assert( pConfig->bLocale );
if( i>1 ){
if( pConfig->abUnindexed[i-2] ){
/* At attempt to insert an fts5_locale() value into an UNINDEXED
** column. Strip the locale away and just bind the text. */
const char *pText = 0;
int nText = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
}else{
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
assert( nBlob>4 );
sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
}
continue;
}
}else if( pConfig->bLocale
&& sqlite3_value_type(pVal)==SQLITE_BLOB
&& i>=2 && pConfig->abUnindexed[i-2]==0
){
/* Inserting a blob into a normal content table with locale=1.
** Add the 4 0x00 byte header. */
int n = sqlite3_value_bytes(pVal);
u8 *pBlob = sqlite3Fts5MallocZero(&rc, n+4);
if( pBlob ){
memcpy(&pBlob[4], sqlite3_value_blob(pVal), n);
rc = sqlite3_bind_blob(pInsert, i, pBlob, n+4, SQLITE_TRANSIENT);
sqlite3_free(pBlob);
}
continue;
}
rc = sqlite3_bind_value(pInsert, i, pVal);
ext/fts5/test/fts5blob.test
+77 -17
View File
@@ -9,6 +9,19 @@
#
#***********************************************************************
#
# This file verifies that:
#
# * blob values may be written to locale=0 tables.
#
# * blob values - other than fts5_locale() values - may not be written
# to locale=0 tables. This is an SQLITE_MISMATCH error
#
# * blob values may be returned by queries on the external-content table
# of a locale=0 table.
#
# * blob values not may be returned by queries on the external-content
# table of a locale=1 table, apart from fts5_locale() blobs. This is an
# SQLITE_MISMATCH error.
#
source [file join [file dirname [info script]] fts5_common.tcl]
@@ -20,11 +33,11 @@ ifcapable !fts5 {
return
}
foreach {tn enc locale} {
1 utf8 0
2 utf8 1
3 utf16 0
4 utf16 1
# Test that blobs may be stored in normal locale=0 tables.
#
foreach {tn enc} {
1 utf8
2 utf16
} {
reset_db
fts5_aux_test_functions db
@@ -32,7 +45,7 @@ foreach {tn enc locale} {
execsql "PRAGMA encoding = $enc"
execsql "
CREATE VIRTUAL TABLE t1 USING fts5(x, y, locale=$locale);
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
"
do_execsql_test 1.$tn.0 {
CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance');
@@ -49,9 +62,6 @@ foreach {tn enc locale} {
3 777 'xyz'
}
set T($enc,$locale) [execsql { SELECT * FROM tt }]
set U($enc,$locale) [execsql { SELECT fts5_test_columntext(t1) FROM t1 }]
do_execsql_test 1.$tn.2 {
DELETE FROM t1 WHERE rowid=2;
DELETE FROM t1 WHERE rowid=1;
@@ -62,27 +72,77 @@ foreach {tn enc locale} {
} {ok}
}
do_test 1.5.1 { set T(utf8,1) } $T(utf8,0)
do_test 1.5.2 { set T(utf16,1) } $T(utf16,0)
do_test 1.6.1 { set U(utf8,1) } $U(utf8,0)
do_test 1.6.2 { set U(utf16,1) } $U(utf16,0)
#--------------------------------------------------------------------------
# Test that a blob may be stored and retrieved in an unindexed column of
# a regular table with locale=1.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
}
breakpoint
do_execsql_test 2.1 {
select rowid, x, quote(y) FROM t1
} {
12 twelve X'0000000041424320444546'
}
#--------------------------------------------------------------------------
# Test that blobs may not be written to any type of table with locale=1
# set. Except, they may be written to UNINDEXED columns.
#
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a, b);
CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1);
CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2);
CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=);
}
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
#--------------------------------------------------------------------------
# Test that fts5_locale() values may not be written to any type of table
# without locale=1 set. Even to an UNINDEXED column.
#
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a, b);
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2);
CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=);
CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED);
}
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.4 {
INSERT INTO x4(rowid, a, b, c)
VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
finish_test
ext/fts5/test/fts5locale.test
+49 -3
View File
@@ -107,7 +107,7 @@ do_execsql_test 2.5 {
do_catchsql_test 2.6 {
INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six'));
} {1 {fts5_locale() may not be used without locale=1}}
} {1 {fts5_locale() requires locale=1}}
do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1}
do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {}
@@ -149,7 +149,7 @@ reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl);
CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1);
CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance);
INSERT INTO c1 VALUES('hello world');
@@ -343,12 +343,58 @@ do_execsql_test 6.0 {
}
do_catchsql_test 6.1 {
INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world'));
} {1 {fts5_locale() may not be used without locale=1}}
} {1 {fts5_locale() requires locale=1}}
do_execsql_test 6.2 {
SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') );
} {text text}
#--------------------------------------------------------------------------
# Test that fts5_locale() works with virtual tables.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
# optimization_control db query-flattener 0
do_execsql_test 7.1 {
CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT);
CREATE VIEW v1 AS
SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1;
CREATE VIRTUAL TABLE ft USING fts5(
bb, tt, locale=1, tokenize=tcl, content=v1
);
INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL);
INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse');
INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second');
}
do_execsql_test 7.2 {
INSERT INTO ft(ft) VALUES('rebuild');
INSERT INTO ft(ft) VALUES('integrity-check');
}
do_execsql_test 7.3 {
SELECT rowid, quote(bb), quote(tt) FROM ft
} {
1 NULL {'one two three'}
2 '7800616263' {'four five six'}
3 '000000007800616263' {'seven eight nine'}
}
do_execsql_test 7.4 { SELECT rowid FROM ft('six'); }
do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2
fts5_aux_test_functions db
do_execsql_test 7.6 {
SELECT fts5_test_columnlocale(ft) FROM ft;
} {
{{} {}} {{} reverse} {{} second}
}
finish_test
manifest
+11 -11
View File
@@ -1,5 +1,5 @@
C Clarify\sthe\srole\sof\sFts5Storage.pSavedRow\sin\sthe\snew\sfeature\son\sthis\sbranch.
D 2024-08-01T17:15:17.507
C Change\sthings\sso\sthat\slocale=1\sis\srequired\sto\swrite\sfts5_locale()\svalues\sto\san\sfts5\stable,\sand\sso\sthat\sblobs\smay\snot\sbe\sstored\sin\sindexed\s(i.e.\snot\sUNINDEXED)\scolumns\sof\sthese\stables.
D 2024-08-02T21:06:13.360
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5
F ext/fts5/fts5Int.h 2eab38c52d12bfa0a1cedb742276035e05634c6365f8defb599f023bb8743559
F ext/fts5/fts5Int.h 64ccc862cbf3be08103ed9435b8670017019f5bd620db4e4317936170008b2b1
F ext/fts5/fts5_aux.c 598c80fc0faabab91c833cdda99f8e36387bd907f4acb0480a19b612a4add93e
F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
F ext/fts5/fts5_config.c 31267fc68f9d16feaa5b5f7efb902a3410f0caa708cc2372c90a77d75890b2a8
F ext/fts5/fts5_config.c b7924194b5dc035d9f0bd82c74564e133ff38aaa2edf86d8a95c23c9c82ba17f
F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad
F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe
F ext/fts5/fts5_main.c bda8d421024191376343a571370a7b2f92fdf5303e32c6dc0d7456bef9ffedd8
F ext/fts5/fts5_storage.c 784c6c4edf676c831ffbaef7b0484dcafe925d3d39049b8ee6b6eeaadae1b042
F ext/fts5/fts5_main.c 79b1d6099d8140afbd930a4e280759728099f584f91e39a8e81f2df0a0e0e839
F ext/fts5/fts5_storage.c 5bf88213ff5911625c142ac332ddba10dcd0869e757f91f2a3d27f27ba595992
F ext/fts5/fts5_tcl.c 93b705cb87633574983161edc5234f9b91ba03f9fecfbd2c5d401a1da6f93aa5
F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
@@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c
F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd
F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef
F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36
F ext/fts5/test/fts5blob.test 6d1cf0c5ba2e6f8f9b4e915d3677c16ead3a79c22dcb386c0b21169a9349718d
F ext/fts5/test/fts5blob.test 2f84ef3591dd3eb0efa686a3615e9ed5c5005462c663c7c72c350692b99f91f5
F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a
F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62
F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897
@@ -185,7 +185,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd
F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1
F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad
F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c
F ext/fts5/test/fts5locale.test bfd8704f9bea963314fcbcf810f08a357ac8035bcb80a2d6170c1e57fa6ad52a
F ext/fts5/test/fts5locale.test 57f4effee98b8c3b8b63f80ce08cc424e68a4ef1b7ce74a91c1e64b2d213053e
F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3
F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082
F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
@@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2
R 65a2f69590c259e395cd85b8f6d4acad
P e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d
R 1b40b0a91a44b0bf6782847dce5b9e1d
U dan
Z cbe99cb860ae582ece42e3b6610aa855
Z 06efbaab9999ac759a0d226937b631dd
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d
c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e