paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixes to built-in tokenizers.

Browse Source 
FossilOrigin-Name: b33fe0dd89f3180c209fa1f9e75d0a7acab12b8e
This commit is contained in:
dan
2014-12-29 11:24:46 +00:00
parent 2a615fa627
commit 5fa3acabf4
10 changed files with 12864 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5.c
+2 -13
View File
@@ -1277,18 +1277,6 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
return rc;
}
static int fts5ApiPoslist(
Fts5Context *pCtx,
int iPhrase,
int *pi,
i64 *piPos
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
const u8 *a; int n; /* Poslist for phrase iPhrase */
n = fts5CsrPoslist(pCsr, iPhrase, &a);
return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}
static int fts5ApiSetAuxdata(
Fts5Context *pCtx, /* Fts5 context */
void *pPtr, /* Pointer to save as auxdata */
@@ -1360,7 +1348,6 @@ static const Fts5ExtensionApi sFts5Api = {
fts5ApiRowid,
fts5ApiColumnText,
fts5ApiColumnSize,
fts5ApiPoslist,
fts5ApiQueryPhrase,
fts5ApiSetAuxdata,
fts5ApiGetAuxdata,
@@ -1682,6 +1669,7 @@ static int fts5CreateTokenizer(
static int fts5FindTokenizer(
fts5_api *pApi, /* Global context (one per db handle) */
const char *zName, /* Name of new function */
void **ppUserData,
fts5_tokenizer *pTokenizer /* Populate this object */
){
Fts5Global *pGlobal = (Fts5Global*)pApi;
@@ -1694,6 +1682,7 @@ static int fts5FindTokenizer(
if( pTok ){
*pTokenizer = pTok->x;
*ppUserData = pTok->pUserData;
}else{
memset(pTokenizer, 0, sizeof(fts5_tokenizer));
rc = SQLITE_ERROR;
ext/fts5/fts5.h
+1 -4
View File
@@ -207,7 +207,6 @@ struct Fts5ExtensionApi {
sqlite3_int64 (*xRowid)(Fts5Context*);
int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);
int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
@@ -216,9 +215,6 @@ struct Fts5ExtensionApi {
void *(*xGetAuxdata)(Fts5Context*, int bClear);
};
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
/*
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
@@ -325,6 +321,7 @@ struct fts5_api {
int (*xFindTokenizer)(
fts5_api *pApi,
const char *zName,
void **ppContext,
fts5_tokenizer *pTokenizer
);
ext/fts5/fts5Int.h
+5 -1
View File
@@ -29,8 +29,9 @@
#define FTS5_DEFAULT_NEARDIST 10
#define FTS5_DEFAULT_RANK "bm25"
/* Name of rank column */
/* Name of rank and rowid columns */
#define FTS5_RANK_NAME "rank"
#define FTS5_ROWID_NAME "rowid"
/**************************************************************************
** Interface to code in fts5.c.
@@ -149,6 +150,9 @@ void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
typedef struct Fts5PoslistReader Fts5PoslistReader;
struct Fts5PoslistReader {
/* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
ext/fts5/fts5_config.c
+268 -156
View File
@@ -21,6 +21,142 @@
/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (128*1024)
static int fts5_iswhitespace(char x){
return (x==' ');
}
static int fts5_isopenquote(char x){
return (x=='"' || x=='\'' || x=='[' || x=='`');
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a white-space character.
*/
static const char *fts5ConfigSkipWhitespace(const char *pIn){
const char *p = pIn;
if( p ){
while( fts5_iswhitespace(*p) ){ p++; }
}
return p;
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
const char *p = pIn;
while( *p && *p!=' ' && *p!=':' && *p!='!' && *p!='@'
&& *p!='#' && *p!='$' && *p!='%' && *p!='^' && *p!='&'
&& *p!='*' && *p!='(' && *p!=')' && *p!='='
){
p++;
}
if( p==pIn ) p = 0;
return p;
}
static int fts5_isdigit(char a){
return (a>='0' && a<='9');
}
static const char *fts5ConfigSkipLiteral(const char *pIn){
const char *p = pIn;
if( p ){
switch( *p ){
case 'n': case 'N':
if( sqlite3_strnicmp("null", p, 4)==0 ){
p = &p[4];
}else{
p = 0;
}
break;
case 'x': case 'X':
p++;
if( *p=='\'' ){
p++;
while( (*p>='a' && *p<='f')
|| (*p>='A' && *p<='F')
|| (*p>='0' && *p<='9')
){
p++;
}
if( *p=='\'' && 0==((p-pIn)%2) ){
p++;
}else{
p = 0;
}
}else{
p = 0;
}
break;
case '\'':
p++;
while( p ){
if( *p=='\'' ){
p++;
if( *p!='\'' ) break;
}
p++;
if( *p==0 ) p = 0;
}
break;
default:
/* maybe a number */
if( *p=='+' || *p=='-' ) p++;
while( fts5_isdigit(*p) ) p++;
/* At this point, if the literal was an integer, the parse is
** finished. Or, if it is a floating point value, it may continue
** with either a decimal point or an 'E' character. */
if( *p=='.' && fts5_isdigit(p[1]) ){
p += 2;
while( fts5_isdigit(*p) ) p++;
}
if( p==pIn ) p = 0;
break;
}
}
return p;
}
static int fts5Dequote(char *z){
char q;
int iIn = 1;
int iOut = 0;
int bRet = 1;
q = z[0];
assert( q=='[' || q=='\'' || q=='"' || q=='`' );
if( q=='[' ) q = ']';
while( z[iIn] ){
if( z[iIn]==q ){
if( z[iIn+1]!=q ){
if( z[iIn+1]=='\0' ) bRet = 0;
break;
}
z[iOut++] = q;
iIn += 2;
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
return bRet;
}
/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters. The conversion is done in-place. If the
@@ -37,27 +173,25 @@
void sqlite3Fts5Dequote(char *z){
char quote; /* Quote character (if any ) */
assert( 0==fts5_iswhitespace(z[0]) );
quote = z[0];
if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
int iIn = 1; /* Index of next byte to read from input */
int iOut = 0; /* Index of next byte to write to output */
/* If the first byte was a '[', then the close-quote character is a ']' */
if( quote=='[' ) quote = ']';
while( ALWAYS(z[iIn]) ){
if( z[iIn]==quote ){
if( z[iIn+1]!=quote ) break;
z[iOut++] = quote;
iIn += 2;
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
fts5Dequote(z);
}
}
/*
** Trim any white-space from the right of nul-terminated string z.
*/
static char *fts5TrimString(char *z){
int n = strlen(z);
while( n>0 && fts5_iswhitespace(z[n-1]) ){
z[--n] = '\0';
}
while( fts5_iswhitespace(*z) ) z++;
return z;
}
/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
@@ -68,15 +202,17 @@ void sqlite3Fts5Dequote(char *z){
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Global *pGlobal,
Fts5Config *pConfig, /* Configuration object to update */
char *zCmd, /* Special command to parse */
char *zArg, /* Argument to parse */
const char *zCmd, /* Special command to parse */
int nCmd, /* Size of zCmd in bytes */
const char *zArg, /* Argument to parse */
char **pzErr /* OUT: Error message */
){
if( sqlite3_stricmp(zCmd, "prefix")==0 ){
if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
int rc = SQLITE_OK;
char *p;
const char *p;
if( pConfig->aPrefix ){
*pzErr = sqlite3_mprintf("multiple prefix=... directives");
rc = SQLITE_ERROR;
@@ -108,6 +244,53 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
int rc = SQLITE_OK;
const char *p = (const char*)zArg;
int nArg = strlen(zArg) + 1;
char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
char *pSpace = pDel;
if( azArg && pSpace ){
if( pConfig->pTok ){
*pzErr = sqlite3_mprintf("multiple tokenize=... directives");
rc = SQLITE_ERROR;
}else{
for(nArg=0; p && *p; nArg++){
const char *p2 = fts5ConfigSkipWhitespace(p);
if( p2 && *p2=='\'' ){
p = fts5ConfigSkipLiteral(p2);
}else{
p = fts5ConfigSkipBareword(p2);
}
if( p ){
memcpy(pSpace, p2, p-p2);
azArg[nArg] = pSpace;
sqlite3Fts5Dequote(pSpace);
pSpace += (p - p2) + 1;
p = fts5ConfigSkipWhitespace(p);
}
}
if( p==0 ){
*pzErr = sqlite3_mprintf("parse error in tokenize directive");
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi
);
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("error in tokenizer constructor");
}
}
}
}
sqlite3_free(azArg);
sqlite3_free(pDel);
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
return SQLITE_ERROR;
}
@@ -133,6 +316,7 @@ static char *fts5Strdup(int *pRc, const char *z){
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
);
@@ -160,6 +344,7 @@ int sqlite3Fts5ConfigParse(
){
int rc = SQLITE_OK; /* Return code */
Fts5Config *pRet; /* New object to return */
int i;
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
@@ -170,44 +355,69 @@ int sqlite3Fts5ConfigParse(
pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
pRet->zDb = fts5Strdup(&rc, azArg[1]);
pRet->zName = fts5Strdup(&rc, azArg[2]);
if( rc==SQLITE_OK ){
if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
rc = SQLITE_ERROR;
}else{
int i;
for(i=3; rc==SQLITE_OK && i<nArg; i++){
char *zDup = fts5Strdup(&rc, azArg[i]);
if( zDup ){
/* Check if this is a special directive - "cmd=arg" */
if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
char *p = zDup;
while( *p && *p!='=' ) p++;
if( *p ){
char *zArg = &p[1];
*p = '\0';
sqlite3Fts5Dequote(zArg);
rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
sqlite3_free(zDup);
zDup = 0;
}
}
if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
rc = SQLITE_ERROR;
}
/* If it is not a special directive, it must be a column name. In
** this case, check that it is not the reserved column name "rank". */
if( zDup ){
sqlite3Fts5Dequote(zDup);
pRet->azCol[pRet->nCol++] = zDup;
if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
rc = SQLITE_ERROR;
for(i=3; rc==SQLITE_OK && i<nArg; i++){
char *zDup = fts5Strdup(&rc, azArg[i]);
if( zDup ){
char *zCol = 0;
int bParseError = 0;
/* Check if this is a quoted column name */
if( fts5_isopenquote(zDup[0]) ){
bParseError = fts5Dequote(zDup);
zCol = zDup;
}else{
char *z = (char*)fts5ConfigSkipBareword(zDup);
if( *z=='\0' ){
zCol = zDup;
}else{
int nCmd = z - zDup;
z = (char*)fts5ConfigSkipWhitespace(z);
if( *z!='=' ){
bParseError = 1;
}else{
z++;
z = fts5TrimString(z);
if( fts5_isopenquote(*z) ){
if( fts5Dequote(z) ) bParseError = 1;
}else{
char *z2 = (char*)fts5ConfigSkipBareword(z);
if( *z2 ) bParseError = 1;
}
if( bParseError==0 ){
rc = fts5ConfigParseSpecial(pGlobal, pRet, zDup, nCmd, z, pzErr);
}
}
}
}
if( bParseError ){
assert( *pzErr==0 );
*pzErr = sqlite3_mprintf("parse error in \"%s\"", zDup);
rc = SQLITE_ERROR;
}else if( zCol ){
if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME)
|| 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME)
){
*pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
rc = SQLITE_ERROR;
}else{
pRet->azCol[pRet->nCol++] = zCol;
zDup = 0;
}
}
sqlite3_free(zDup);
}
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (simple) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
@@ -309,106 +519,6 @@ int sqlite3Fts5Tokenize(
return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a white-space character.
*/
static const char *fts5ConfigSkipWhitespace(const char *pIn){
const char *p = pIn;
if( p ){
while( *p==' ' ){ p++; }
}
return p;
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
const char *p = pIn;
while( *p && *p!=' ' && *p!=':' && *p!='!' && *p!='@'
&& *p!='#' && *p!='$' && *p!='%' && *p!='^' && *p!='&'
&& *p!='*' && *p!='(' && *p!=')'
){
p++;
}
if( p==pIn ) p = 0;
return p;
}
static int fts5_isdigit(char a){
return (a>='0' && a<='9');
}
static const char *fts5ConfigSkipLiteral(const char *pIn){
const char *p = pIn;
if( p ){
switch( *p ){
case 'n': case 'N':
if( sqlite3_strnicmp("null", p, 4)==0 ){
p = &p[4];
}else{
p = 0;
}
break;
case 'x': case 'X':
p++;
if( *p=='\'' ){
p++;
while( (*p>='a' && *p<='f')
|| (*p>='A' && *p<='F')
|| (*p>='0' && *p<='9')
){
p++;
}
if( *p=='\'' && 0==((p-pIn)%2) ){
p++;
}else{
p = 0;
}
}else{
p = 0;
}
break;
case '\'':
p++;
while( p ){
if( *p=='\'' ){
p++;
if( *p!='\'' ) break;
}
p++;
if( *p==0 ) p = 0;
}
break;
default:
/* maybe a number */
if( *p=='+' || *p=='-' ) p++;
while( fts5_isdigit(*p) ) p++;
/* At this point, if the literal was an integer, the parse is
** finished. Or, if it is a floating point value, it may continue
** with either a decimal point or an 'E' character. */
if( *p=='.' && fts5_isdigit(p[1]) ){
p += 2;
while( fts5_isdigit(*p) ) p++;
}
break;
}
}
return p;
}
/*
** Argument pIn points to the first character in what is expected to be
** a comma-separated list of SQL literals followed by a ')' character.
@@ -476,12 +586,14 @@ static int fts5ConfigParseRank(
const char *pArgs;
p = fts5ConfigSkipWhitespace(p);
pArgs = p;
p = fts5ConfigSkipArgs(p);
if( p==0 ){
rc = SQLITE_ERROR;
}else if( p!=pArgs ){
zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
if( *p!=')' ){
p = fts5ConfigSkipArgs(p);
if( p==0 ){
rc = SQLITE_ERROR;
}else if( p!=pArgs ){
zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
}
}
}
ext/fts5/fts5_tcl.c
+332 -23
View File
@@ -12,6 +12,7 @@
**
*/
#ifdef SQLITE_TEST
#include "fts5.h"
#include <tcl.h>
@@ -42,9 +43,47 @@ static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){
}
return TCL_ERROR;
}
/* End of code that accesses the SqliteDb struct.
**************************************************************************/
static int f5tDbAndApi(
Tcl_Interp *interp,
Tcl_Obj *pObj,
sqlite3 **ppDb,
fts5_api **ppApi
){
sqlite3 *db = 0;
int rc = f5tDbPointer(interp, pObj, &db);
if( rc!=TCL_OK ){
return TCL_ERROR;
}else{
sqlite3_stmt *pStmt = 0;
fts5_api *pApi = 0;
rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
if( SQLITE_ROW==sqlite3_step(pStmt) ){
const void *pPtr = sqlite3_column_blob(pStmt, 0);
memcpy((void*)&pApi, pPtr, sizeof(pApi));
}
if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
*ppDb = db;
*ppApi = pApi;
}
return TCL_OK;
}
typedef struct F5tFunction F5tFunction;
struct F5tFunction {
Tcl_Interp *interp;
@@ -451,7 +490,6 @@ static int f5tCreateFunction(
char *zName;
Tcl_Obj *pScript;
sqlite3 *db = 0;
sqlite3_stmt *pStmt = 0;
fts5_api *pApi = 0;
F5tFunction *pCtx = 0;
int rc;
@@ -460,27 +498,10 @@ static int f5tCreateFunction(
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
return TCL_ERROR;
}
if( f5tDbPointer(interp, objv[1], &db) ){
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
zName = Tcl_GetString(objv[2]);
pScript = objv[3];
rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
if( SQLITE_ROW==sqlite3_step(pStmt) ){
const void *pPtr = sqlite3_column_blob(pStmt, 0);
memcpy((void*)&pApi, pPtr, sizeof(pApi));
}
if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
return TCL_ERROR;
}
pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction));
pCtx->interp = interp;
pCtx->pScript = pScript;
@@ -497,6 +518,284 @@ static int f5tCreateFunction(
return TCL_OK;
}
static int xTokenizeCb2(
void *pCtx,
const char *zToken, int nToken,
int iStart, int iEnd, int iPos
){
Tcl_Obj *pRet = (Tcl_Obj*)pCtx;
Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iStart));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iEnd));
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
return SQLITE_OK;
}
/*
** sqlite3_fts5_tokenize DB TOKENIZER TEXT
**
** Description...
*/
static int f5tTokenize(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
char *zName;
char *zText;
int nText;
sqlite3 *db = 0;
fts5_api *pApi = 0;
Fts5Tokenizer *pTok = 0;
fts5_tokenizer tokenizer;
Tcl_Obj *pRet = 0;
void *pUserdata;
int rc;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME TEXT");
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
zName = Tcl_GetString(objv[2]);
zText = Tcl_GetStringFromObj(objv[3], &nText);
rc = pApi->xFindTokenizer(pApi, zName, &pUserdata, &tokenizer);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "no such tokenizer: ", zName, 0);
return TCL_ERROR;
}
rc = tokenizer.xCreate(pUserdata, 0, 0, &pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
return TCL_ERROR;
}
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
rc = tokenizer.xTokenize(pTok, pRet, zText, nText, xTokenizeCb2);
tokenizer.xDelete(pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
Tcl_DecrRefCount(pRet);
return TCL_ERROR;
}
Tcl_SetObjResult(interp, pRet);
Tcl_DecrRefCount(pRet);
return TCL_OK;
}
/*************************************************************************
** Start of tokenizer wrapper.
*/
typedef struct F5tTokenizerContext F5tTokenizerContext;
typedef struct F5tTokenizerCb F5tTokenizerCb;
typedef struct F5tTokenizerModule F5tTokenizerModule;
typedef struct F5tTokenizerModule F5tTokenizerInstance;
struct F5tTokenizerContext {
void *pCtx;
int (*xToken)(void*, const char*, int, int, int, int);
};
struct F5tTokenizerModule {
Tcl_Interp *interp;
Tcl_Obj *pScript;
F5tTokenizerContext *pContext;
};
static int f5tTokenizerCreate(
void *pCtx,
const char **azArg,
int nArg,
Fts5Tokenizer **ppOut
){
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
Tcl_Obj *pEval;
int rc = TCL_OK;
int i;
pEval = Tcl_DuplicateObj(pMod->pScript);
Tcl_IncrRefCount(pEval);
for(i=0; rc==TCL_OK && i<nArg; i++){
Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1);
rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj);
}
if( rc==TCL_OK ){
rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY);
}
Tcl_DecrRefCount(pEval);
if( rc==TCL_OK ){
F5tTokenizerInstance *pInst = ckalloc(sizeof(F5tTokenizerInstance));
memset(pInst, 0, sizeof(F5tTokenizerInstance));
pInst->interp = pMod->interp;
pInst->pScript = Tcl_GetObjResult(pMod->interp);
pInst->pContext = pMod->pContext;
Tcl_IncrRefCount(pInst->pScript);
*ppOut = (Fts5Tokenizer*)pInst;
}
return rc;
}
static void f5tTokenizerDelete(Fts5Tokenizer *p){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
Tcl_DecrRefCount(pInst->pScript);
ckfree(pInst);
}
static int f5tTokenizerTokenize(
Fts5Tokenizer *p,
void *pCtx,
const char *pText, int nText,
int (*xToken)(void*, const char*, int, int, int, int)
){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
void *pOldCtx;
int (*xOldToken)(void*, const char*, int, int, int, int);
Tcl_Obj *pEval;
int rc;
pOldCtx = pInst->pContext->pCtx;
xOldToken = pInst->pContext->xToken;
pEval = Tcl_DuplicateObj(pInst->pScript);
Tcl_IncrRefCount(pEval);
rc = Tcl_ListObjAppendElement(
pInst->interp, pEval, Tcl_NewStringObj(pText, nText)
);
if( rc==TCL_OK ){
rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
}
Tcl_DecrRefCount(pEval);
pInst->pContext->pCtx = pOldCtx;
pInst->pContext->xToken = xOldToken;
return rc;
}
extern const char *sqlite3ErrName(int);
/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
int iStart;
int iEnd;
int iPos;
int nToken;
char *zToken;
int rc;
assert( p );
if( objc!=5 ){
Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END POS");
return TCL_ERROR;
}
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_token may only be used by tokenizer callback", 0
);
return TCL_ERROR;
}
zToken = Tcl_GetStringFromObj(objv[1], &nToken);
if( Tcl_GetIntFromObj(interp, objv[2], &iStart)
|| Tcl_GetIntFromObj(interp, objv[3], &iEnd)
|| Tcl_GetIntFromObj(interp, objv[4], &iPos)
){
return TCL_ERROR;
}
rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd, iPos);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return TCL_OK;
}
static void f5tDelTokenizer(void *pCtx){
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
Tcl_DecrRefCount(pMod->pScript);
ckfree(pMod);
}
/*
** sqlite3_fts5_create_tokenizer DB NAME SCRIPT
**
** Register a tokenizer named NAME implemented by script SCRIPT. When
** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer
** arguments are appended to SCRIPT and the result executed.
**
** The value returned by (SCRIPT + args) is itself a tcl script. This
** script - call it SCRIPT2 - is executed to tokenize text using the
** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize
** text SCRIPT2 is invoked with a single argument appended to it - the
** text to tokenize.
**
** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
** token within the tokenized text.
*/
static int f5tCreateTokenizer(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData;
sqlite3 *db;
fts5_api *pApi;
char *zName;
Tcl_Obj *pScript;
fts5_tokenizer t;
F5tTokenizerModule *pMod;
int rc;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
return TCL_ERROR;
}
zName = Tcl_GetString(objv[2]);
pScript = objv[3];
t.xCreate = f5tTokenizerCreate;
t.xTokenize = f5tTokenizerTokenize;
t.xDelete = f5tTokenizerDelete;
pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
pMod->interp = interp;
pMod->pScript = pScript;
pMod->pContext = pContext;
Tcl_IncrRefCount(pScript);
rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
return TCL_ERROR;
}
return TCL_OK;
}
static void xF5tFree(ClientData clientData){
ckfree(clientData);
}
/*
** Entry point.
*/
@@ -504,17 +803,27 @@ int Fts5tcl_Init(Tcl_Interp *interp){
static struct Cmd {
char *zName;
Tcl_ObjCmdProc *xProc;
void *clientData;
int bTokenizeCtx;
} aCmd[] = {
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 }
{ "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
{ "sqlite3_fts5_token", f5tTokenizerReturn, 1 },
{ "sqlite3_fts5_tokenize", f5tTokenize, 0 },
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 }
};
int i;
F5tTokenizerContext *pContext;
pContext = ckalloc(sizeof(F5tTokenizerContext));
memset(pContext, 0, sizeof(*pContext));
for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
struct Cmd *p = &aCmd[i];
Tcl_CreateObjCommand(interp, p->zName, p->xProc, p->clientData, 0);
void *pCtx = 0;
if( p->bTokenizeCtx ) pCtx = (void*)pContext;
Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
}
return TCL_OK;
}
#endif
ext/fts5/fts5_tokenize.c
+357 -3
View File
@@ -12,6 +12,8 @@
*/
#include "fts5.h"
#include <string.h>
#include <assert.h>
/*
@@ -115,16 +117,368 @@ static int fts5SimpleTokenize(
return rc;
}
/**************************************************************************
** Start of porter2 stemmer implementation.
*/
/* Any tokens larger than this (in bytes) are passed through without
** stemming. */
#define FTS5_PORTER_MAX_TOKEN 64
typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
fts5_tokenizer tokenizer; /* Parent tokenizer module */
Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};
/*
** Delete a "porter" tokenizer.
*/
static void fts5PorterDelete(Fts5Tokenizer *pTok){
if( pTok ){
PorterTokenizer *p = (PorterTokenizer*)pTok;
if( p->pTokenizer ){
p->tokenizer.xDelete(p->pTokenizer);
}
sqlite3_free(p);
}
}
/*
** Create a "porter" tokenizer.
*/
static int fts5PorterCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
fts5_api *pApi = (fts5_api*)pCtx;
int rc = SQLITE_OK;
PorterTokenizer *pRet;
void *pUserdata = 0;
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, "simple", &pUserdata, &pRet->tokenizer);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
}
if( rc!=SQLITE_OK ){
fts5PorterDelete((Fts5Tokenizer*)pRet);
pRet = 0;
}
*ppOut = (Fts5Tokenizer*)pRet;
return rc;
}
typedef struct PorterContext PorterContext;
struct PorterContext {
void *pCtx;
int (*xToken)(void*, const char*, int, int, int, int);
char *aBuf;
};
typedef struct PorterRule PorterRule;
struct PorterRule {
const char *zSuffix;
int nSuffix;
int (*xCond)(char *zStem, int nStem);
const char *zOutput;
int nOutput;
};
static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
int ret = -1;
int nBuf = *pnBuf;
PorterRule *p;
for(p=aRule; p->zSuffix; p++){
assert( strlen(p->zSuffix)==p->nSuffix );
assert( strlen(p->zOutput)==p->nOutput );
if( nBuf<p->nSuffix ) continue;
if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
}
if( p->zSuffix ){
int nStem = nBuf - p->nSuffix;
if( p->xCond==0 || p->xCond(aBuf, nStem) ){
memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
*pnBuf = nStem + p->nOutput;
ret = p - aRule;
}
}
return ret;
}
static int fts5PorterIsVowel(char c, int bYIsVowel){
return (
c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
);
}
static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
int i;
int bCons = bPrevCons;
/* Scan for a vowel */
for(i=0; i<nStem; i++){
if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
}
/* Scan for a consonent */
for(i++; i<nStem; i++){
if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
}
return 0;
}
/* porter rule condition: (m > 0) */
static int fts5Porter_MGt0(char *zStem, int nStem){
return !!fts5PorterGobbleVC(zStem, nStem, 0);
}
/* porter rule condition: (m > 1) */
static int fts5Porter_MGt1(char *zStem, int nStem){
int n;
n = fts5PorterGobbleVC(zStem, nStem, 0);
if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
return 1;
}
return 0;
}
/* porter rule condition: (m = 1) */
static int fts5Porter_MEq1(char *zStem, int nStem){
int n;
n = fts5PorterGobbleVC(zStem, nStem, 0);
if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
return 1;
}
return 0;
}
/* porter rule condition: (*o) */
static int fts5Porter_Ostar(char *zStem, int nStem){
if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
return 0;
}else{
int i;
int mask = 0;
int bCons = 0;
for(i=0; i<nStem; i++){
bCons = !fts5PorterIsVowel(zStem[i], bCons);
assert( bCons==0 || bCons==1 );
mask = (mask << 1) + bCons;
}
return ((mask & 0x0007)==0x0005);
}
}
/* porter rule condition: (m > 1 and (*S or *T)) */
static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
return nStem>0
&& (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
&& fts5Porter_MGt1(zStem, nStem);
}
/* porter rule condition: (*v*) */
static int fts5Porter_Vowel(char *zStem, int nStem){
int i;
for(i=0; i<nStem; i++){
if( fts5PorterIsVowel(zStem[i], i>0) ){
return 1;
}
}
return 0;
}
static int fts5PorterCb(
void *pCtx,
const char *pToken,
int nToken,
int iStart,
int iEnd,
int iPos
){
PorterContext *p = (PorterContext*)pCtx;
PorterRule aStep1A[] = {
{ "sses", 4, 0, "ss", 2 },
{ "ies", 3, 0, "i", 1 },
{ "ss", 2, 0, "ss", 2 },
{ "s", 1, 0, "", 0 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1B[] = {
{ "eed", 3, fts5Porter_MGt0, "ee", 2 },
{ "ed", 2, fts5Porter_Vowel, "", 0 },
{ "ing", 3, fts5Porter_Vowel, "", 0 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1B2[] = {
{ "at", 2, 0, "ate", 3 },
{ "bl", 2, 0, "ble", 3 },
{ "iz", 2, 0, "ize", 3 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1C[] = {
{ "y", 1, fts5Porter_Vowel, "i", 1 },
{ 0, 0, 0, 0 }
};
PorterRule aStep2[] = {
{ "ational", 7, fts5Porter_MGt0, "ate", 3},
{ "tional", 6, fts5Porter_MGt0, "tion", 4},
{ "enci", 4, fts5Porter_MGt0, "ence", 4},
{ "anci", 4, fts5Porter_MGt0, "ance", 4},
{ "izer", 4, fts5Porter_MGt0, "ize", 3},
{ "logi", 4, fts5Porter_MGt0, "log", 3}, /* added post 1979 */
{ "bli", 3, fts5Porter_MGt0, "ble", 3}, /* modified post 1979 */
{ "alli", 4, fts5Porter_MGt0, "al", 2},
{ "entli", 5, fts5Porter_MGt0, "ent", 3},
{ "eli", 3, fts5Porter_MGt0, "e", 1},
{ "ousli", 5, fts5Porter_MGt0, "ous", 3},
{ "ization", 7, fts5Porter_MGt0, "ize", 3},
{ "ation", 5, fts5Porter_MGt0, "ate", 3},
{ "ator", 4, fts5Porter_MGt0, "ate", 3},
{ "alism", 5, fts5Porter_MGt0, "al", 2},
{ "iveness", 7, fts5Porter_MGt0, "ive", 3},
{ "fulness", 7, fts5Porter_MGt0, "ful", 3},
{ "ousness", 7, fts5Porter_MGt0, "ous", 3},
{ "aliti", 5, fts5Porter_MGt0, "al", 2},
{ "iviti", 5, fts5Porter_MGt0, "ive", 3},
{ "biliti", 6, fts5Porter_MGt0, "ble", 3},
{ 0, 0, 0, 0 }
};
PorterRule aStep3[] = {
{ "icate", 5, fts5Porter_MGt0, "ic", 2},
{ "ative", 5, fts5Porter_MGt0, "", 0},
{ "alize", 5, fts5Porter_MGt0, "al", 2},
{ "iciti", 5, fts5Porter_MGt0, "ic", 2},
{ "ical", 4, fts5Porter_MGt0, "ic", 2},
{ "ful", 3, fts5Porter_MGt0, "", 0},
{ "ness", 4, fts5Porter_MGt0, "", 0},
{ 0, 0, 0, 0 }
};
PorterRule aStep4[] = {
{ "al", 2, fts5Porter_MGt1, "", 0},
{ "ance", 4, fts5Porter_MGt1, "", 0},
{ "ence", 4, fts5Porter_MGt1, "", 0},
{ "er", 2, fts5Porter_MGt1, "", 0},
{ "ic", 2, fts5Porter_MGt1, "", 0},
{ "able", 4, fts5Porter_MGt1, "", 0},
{ "ible", 4, fts5Porter_MGt1, "", 0},
{ "ant", 3, fts5Porter_MGt1, "", 0},
{ "ement", 5, fts5Porter_MGt1, "", 0},
{ "ment", 4, fts5Porter_MGt1, "", 0},
{ "ent", 3, fts5Porter_MGt1, "", 0},
{ "ion", 3, fts5Porter_MGt1_and_S_or_T, "", 0},
{ "ou", 2, fts5Porter_MGt1, "", 0},
{ "ism", 3, fts5Porter_MGt1, "", 0},
{ "ate", 3, fts5Porter_MGt1, "", 0},
{ "iti", 3, fts5Porter_MGt1, "", 0},
{ "ous", 3, fts5Porter_MGt1, "", 0},
{ "ive", 3, fts5Porter_MGt1, "", 0},
{ "ize", 3, fts5Porter_MGt1, "", 0},
{ 0, 0, 0, 0 }
};
char *aBuf;
int nBuf;
int n;
if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through;
aBuf = p->aBuf;
nBuf = nToken;
memcpy(aBuf, pToken, nBuf);
/* Step 1. */
fts5PorterApply(aBuf, &nBuf, aStep1A);
n = fts5PorterApply(aBuf, &nBuf, aStep1B);
if( n==1 || n==2 ){
if( fts5PorterApply(aBuf, &nBuf, aStep1B2)<0 ){
char c = aBuf[nBuf-1];
if( fts5PorterIsVowel(c, 0)==0
&& c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
){
nBuf--;
}else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
aBuf[nBuf++] = 'e';
}
}
}
fts5PorterApply(aBuf, &nBuf, aStep1C);
/* Steps 2 through 4. */
fts5PorterApply(aBuf, &nBuf, aStep2);
fts5PorterApply(aBuf, &nBuf, aStep3);
fts5PorterApply(aBuf, &nBuf, aStep4);
/* Step 5a. */
if( nBuf>0 && aBuf[nBuf-1]=='e' ){
if( fts5Porter_MGt1(aBuf, nBuf-1)
|| (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
){
nBuf--;
}
}
/* Step 5b. */
if( nBuf>1 && aBuf[nBuf-1]=='l'
&& aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
){
nBuf--;
}
return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos);
pass_through:
return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos);
}
/*
** Tokenize using the porter tokenizer.
*/
static int fts5PorterTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
const char *pText, int nText,
int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
PorterContext sCtx;
sCtx.xToken = xToken;
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer.xTokenize(
p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb
);
}
/*
** Register all built-in tokenizers with FTS5.
*/
int sqlite3Fts5TokenizerInit(fts5_api *pApi){
struct BuiltinTokenizer {
const char *zName;
void *pUserData;
fts5_tokenizer x;
} aBuiltin[] = {
{ "simple", 0, { fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize } }
{ "porter", { fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize } },
{ "simple", { fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize } }
};
int rc = SQLITE_OK; /* Return code */
@@ -133,7 +487,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
rc = pApi->xCreateTokenizer(pApi,
aBuiltin[i].zName,
&aBuiltin[i].pUserData,
(void*)pApi,
&aBuiltin[i].x,
0
);
ext/fts5/fts5porter.test
+11803
View File
File diff suppressed because it is too large Load Diff
ext/fts5/fts5tokenizer.test
+82
View File
@@ -0,0 +1,82 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the fts5 tokenizers
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set testprefix fts5tokenizer
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
DROP TABLE ft1;
}
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize='porter');
DROP TABLE ft1;
}
do_execsql_test 1.2 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = porter);
DROP TABLE ft1;
}
do_execsql_test 1.3 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter');
DROP TABLE ft1;
}
do_execsql_test 1.4 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter simple');
DROP TABLE ft1;
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 {
SELECT rowid FROM ft1 WHERE ft1 MATCH 'database embedding'
} 1
proc tcl_create {args} {
set ::targs $args
error "failed"
}
sqlite3_fts5_create_tokenizer db tcl tcl_create
foreach {tn directive expected} {
1 {tokenize='tcl a b c'} {a b c}
2 {tokenize='tcl ''d'' ''e'' ''f'''} {d e f}
3 {tokenize="tcl 'g' 'h' 'i'"} {g h i}
4 {tokenize = tcl} {}
} {
do_catchsql_test 3.$tn.1 "
CREATE VIRTUAL TABLE ft2 USING fts5(x, $directive)
" {1 {error in tokenizer constructor}}
do_test 3.$tn.2 { set ::targs } $expected
}
do_catchsql_test 4.1 {
CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc);
} {1 {parse error in "tokenize = tcl abc"}}
do_catchsql_test 4.2 {
CREATE VIRTUAL TABLE ft2 USING fts5(x y)
} {1 {parse error in "x y"}}
finish_test
manifest
+13 -11
View File
@@ -1,5 +1,5 @@
C Fix\sthe\sfts5\sbm25()\sfunction\sso\sthat\sit\smatches\sthe\sdocumentation.
D 2014-12-23T19:18:34.426
C Fixes\sto\sbuilt-in\stokenizers.
D 2014-12-29T11:24:46.773
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,20 +104,22 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
F ext/fts5/fts5.c 6dc8a8504d84aef13d922db06faa8fbcf8c11424
F ext/fts5/fts5.h 7598f4b55b888890650829124717874973c52649
F ext/fts5/fts5Int.h 36054b1dfc4881a9b94f945b348ab6cc01c0c7a5
F ext/fts5/fts5.c 37e124e24e5860f9842e5f3ee22129a786c0fd74
F ext/fts5/fts5.h 4f9d2c477c0ee1907164642471329a82cb6b203b
F ext/fts5/fts5Int.h b5dfed6a1b256ff21d11898f14ab337205844469
F ext/fts5/fts5_aux.c 445e54031ff94174673f4f5aac6c064df20a2a6b
F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1
F ext/fts5/fts5_config.c 5caeb4e77680d635be25b899f97a29cf26fb45ce
F ext/fts5/fts5_config.c 73774e37a99218833b767f96bb5af35ebe43b77c
F ext/fts5/fts5_expr.c 27d3d2deebae277c34ae2bb3d501dd879c442ba5
F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
F ext/fts5/fts5_index.c 4a8e8535b4303400ddb5f6fb08152da0d88ebf6f
F ext/fts5/fts5_storage.c 13794781977c9a624eb8bd7b9509de241e405853
F ext/fts5/fts5_tcl.c 4392e74421d24cc37c370732e8b48217cd2c1777
F ext/fts5/fts5_tokenize.c 8360c0d1ae0d4696f3cc13f7c67a2db6011cdc5b
F ext/fts5/fts5_tcl.c ce11e46589986b957b89809aabd3936d898d501b
F ext/fts5/fts5_tokenize.c 5d6e785345b0d87d174fcc0653bfacd0d9fd7f2e
F ext/fts5/fts5auxdata.test 3844d0f098441cedf75b9cc96d5e6e94d1a3bef4
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/fts5porter.test bb0fba17ce825527addec33fdb12b0062aabd588
F ext/fts5/fts5tokenizer.test 09096cebc17a41650b52eec6e45c9a29923bfdcd w ext/fts5/fts5porter2.test
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -1210,7 +1212,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P ca5d44042aa7461dcc8b700b0763df4df9d4a891
R 62a7bacc31f6964d59aeb316de8e27d6
P 1ac7a8d0af9a71ddf6a1421033dcb9fa67c6120c
R c89483d1ebbc9d631471515ddc2ac098
U dan
Z 38f208724902306f1118acd017f9d3d1
Z 32745924be785a3126a9785b730c9992
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
1ac7a8d0af9a71ddf6a1421033dcb9fa67c6120c
b33fe0dd89f3180c209fa1f9e75d0a7acab12b8e