Add tests for modifying the docid and languageid fields of an fts table with a non-zero languageid_bits field.

FossilOrigin-Name: 949425d467947c3905cca3e08750fbaf2ef645db

ext/fts3/fts3.c

@@ -1081,6 +1081,7 @@ static int fts3InitVtab(
   char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
   char *zContent = 0;             /* content=? parameter (or NULL) */
   char *zLanguageid = 0;          /* languageid=? parameter (or NULL) */
+  char *zLanguageidBits = 0;      /* languageid_bits=? parameter (or NULL) */
 
   assert( strlen(argv[0])==4 );
   assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
@@ -1125,13 +1126,14 @@ static int fts3InitVtab(
         const char *zOpt;
         int nOpt;
       } aFts4Opt[] = {
-        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
-        { "prefix",      6 },     /* 1 -> PREFIX */
-        { "compress",    8 },     /* 2 -> COMPRESS */
-        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
-        { "order",       5 },     /* 4 -> ORDER */
-        { "content",     7 },     /* 5 -> CONTENT */
-        { "languageid", 10 }      /* 6 -> LANGUAGEID */
+        { "matchinfo",        9 },     /* 0 -> MATCHINFO */
+        { "prefix",           6 },     /* 1 -> PREFIX */
+        { "compress",         8 },     /* 2 -> COMPRESS */
+        { "uncompress",      10 },     /* 3 -> UNCOMPRESS */
+        { "order",            5 },     /* 4 -> ORDER */
+        { "content",          7 },     /* 5 -> CONTENT */
+        { "languageid",      10 },     /* 6 -> LANGUAGEID */
+        { "languageid_bits", 15 }      /* 7 -> LANGUAGEID_BITS */
       };
 
       int iOpt;
@@ -1197,6 +1199,13 @@ static int fts3InitVtab(
               zLanguageid = zVal;
               zVal = 0;
               break;
+
+            case 7:              /* LANGUAGEID_BITS */
+              assert( iOpt==7 );
+              sqlite3_free(zLanguageidBits);
+              zLanguageidBits = zVal;
+              zVal = 0;
+              break;
           }
         }
         sqlite3_free(zVal);
@@ -1292,6 +1301,15 @@ static int fts3InitVtab(
   p->zLanguageid = zLanguageid;
   zContent = 0;
   zLanguageid = 0;
+  if( zLanguageidBits && p->zLanguageid && p->zContentTbl==0 ){
+    p->nLanguageidBits = atoi(zLanguageidBits);
+    if( p->nLanguageidBits>30 || p->nLanguageidBits<0 ){
+      rc = SQLITE_ERROR;
+      *pzErr = sqlite3_mprintf("languageid_bits parameter out of range");
+      goto fts3_init_out;
+    }
+  }
+
   TESTONLY( p->inTransaction = -1 );
   TESTONLY( p->mxSavepoint = -1 );
 
@@ -1365,6 +1383,7 @@ fts3_init_out:
   sqlite3_free(zUncompress);
   sqlite3_free(zContent);
   sqlite3_free(zLanguageid);
+  sqlite3_free(zLanguageidBits);
   sqlite3_free((void *)aCol);
   if( rc!=SQLITE_OK ){
     if( p ){
@@ -1426,12 +1445,12 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   pInfo->estimatedCost = 500000;
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+    int iCol = pCons->iColumn;
     if( pCons->usable==0 ) continue;
 
     /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
-    if( iCons<0 
-     && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
-     && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
+    if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
+     && (iCol<0 || (iCol==p->nColumn+1 && p->nLanguageidBits==0))
     ){
       pInfo->idxNum = FTS3_DOCID_SEARCH;
       pInfo->estimatedCost = 1.0;
@@ -1474,9 +1493,13 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
   ** docid) order. Both ascending and descending are possible. 
   */
+  assert( pInfo->orderByConsumed==0 );
   if( pInfo->nOrderBy==1 ){
     struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
-    if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
+    if( pOrder->iColumn<0 || (
+          (pOrder->iColumn==p->nColumn+1)
+       && (pInfo->idxNum>=FTS3_FULLTEXT_SEARCH || p->nLanguageidBits==0)
+    )){
       if( pOrder->desc ){
         pInfo->idxStr = "DESC";
       }else{
@@ -3062,10 +3085,10 @@ static int fts3ColumnMethod(
   assert( iCol>=0 && iCol<=p->nColumn+2 );
 
   if( iCol==p->nColumn+1 ){
-    /* This call is a request for the "docid" column. Since "docid" is an 
-    ** alias for "rowid", use the xRowid() method to obtain the value.
-    */
-    sqlite3_result_int64(pCtx, pCsr->iPrevId);
+    /* This call is a request for the "docid" column. The value currently
+    ** stored in pCsr->iPrevId is a rowid. Transform this to a docid and
+    ** return it.  */
+    sqlite3_result_int64(pCtx, sqlite3Fts3RowidToDocid(p, pCsr->iPrevId));
   }else if( iCol==p->nColumn ){
     /* The extra column whose name is the same as the table.
     ** Return a blob which is a pointer to the cursor.  */

ext/fts3/fts3Int.h

@@ -209,6 +209,7 @@ struct Fts3Table {
   sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   char *zContentTbl;              /* content=xxx option, or NULL */
   char *zLanguageid;              /* languageid=xxx option, or NULL */
+  int nLanguageidBits;            /* languageid_bits=N option, or 0 */
   u8 bAutoincrmerge;              /* True if automerge=1 */
   u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
 
@@ -421,8 +422,9 @@ struct Fts3Expr {
 #define FTSQUERY_OR     4
 #define FTSQUERY_PHRASE 5
 
-
 /* fts3_write.c */
+i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid);
+i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid);
 int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
 int sqlite3Fts3PendingTermsFlush(Fts3Table *);
 void sqlite3Fts3PendingTermsClear(Fts3Table *);

ext/fts3/fts3_write.c

@@ -487,6 +487,24 @@ static void fts3SqlExec(
   *pRC = rc;
 }
 
+static void fts3SqlExecI64(
+  int *pRC,                /* Result code */
+  Fts3Table *p,            /* The FTS3 table */
+  int eStmt,               /* Index of statement to evaluate */
+  i64 iVal
+){
+  sqlite3_stmt *pStmt;
+  int rc;
+  if( *pRC ) return;
+  rc = fts3SqlStmt(p, eStmt, &pStmt, 0); 
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pStmt, 1, iVal);
+    sqlite3_step(pStmt);
+    rc = sqlite3_reset(pStmt);
+  }
+  *pRC = rc;
+}
+
 
 /*
 ** This function ensures that the caller has obtained an exclusive 
@@ -927,20 +945,23 @@ static int fts3InsertTerms(
 static int fts3InsertData(
   Fts3Table *p,                   /* Full-text table */
   sqlite3_value **apVal,          /* Array of values to insert */
-  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
+  sqlite3_int64 *piRowid,         /* OUT: Rowid for row just inserted */
+  i64 iRowid                      /* Explicit rowid, if piRowid==NULL */
 ){
   int rc;                         /* Return code */
   sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */
 
   if( p->zContentTbl ){
-    sqlite3_value *pRowid = apVal[p->nColumn+3];
+    sqlite3_value *pRowid;
+    assert( p->nLanguageidBits==0 && piRowid );
+    pRowid = apVal[p->nColumn+3];
     if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
       pRowid = apVal[1];
     }
     if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
       return SQLITE_CONSTRAINT;
     }
-    *piDocid = sqlite3_value_int64(pRowid);
+    *piRowid = sqlite3_value_int64(pRowid);
     return SQLITE_OK;
   }
 
@@ -953,11 +974,16 @@ static int fts3InsertData(
   ** defined columns in the FTS3 table, plus one for the docid field.
   */
   rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
-  if( rc==SQLITE_OK && p->zLanguageid ){
-    rc = sqlite3_bind_int(
-        pContentInsert, p->nColumn+2, 
-        sqlite3_value_int(apVal[p->nColumn+4])
-    );
+  if( rc==SQLITE_OK ){
+    if( piRowid==0 ){
+      sqlite3_bind_int64(pContentInsert, 1, iRowid);
+    }
+    if( p->zLanguageid ){
+      rc = sqlite3_bind_int(
+          pContentInsert, p->nColumn+2, 
+          sqlite3_value_int(apVal[p->nColumn+4])
+      );
+    }
   }
   if( rc!=SQLITE_OK ) return rc;
 
@@ -971,24 +997,25 @@ static int fts3InsertData(
   ** In FTS3, this is an error. It is an error to specify non-NULL values
   ** for both docid and some other rowid alias.
   */
-  if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
-    if( SQLITE_NULL==sqlite3_value_type(apVal[0])
-     && SQLITE_NULL!=sqlite3_value_type(apVal[1])
-    ){
-      /* A rowid/docid conflict. */
-      return SQLITE_ERROR;
-    }
+  assert( p->nLanguageidBits==0 || piRowid==0
+      || sqlite3_value_type(apVal[1])!=SQLITE_NULL 
+  );
+  if( piRowid && p->nLanguageidBits==0 
+   && SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) 
+  ){
     rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
     if( rc!=SQLITE_OK ) return rc;
   }
 
-  /* Execute the statement to insert the record. Set *piDocid to the 
+  /* Execute the statement to insert the record. Set *pRowid to the 
   ** new docid value. 
   */
   sqlite3_step(pContentInsert);
   rc = sqlite3_reset(pContentInsert);
 
-  *piDocid = sqlite3_last_insert_rowid(p->db);
+  if( piRowid ){
+    *piRowid = sqlite3_last_insert_rowid(p->db);
+  }
   return rc;
 }
 
@@ -1036,7 +1063,7 @@ static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){
 static void fts3DeleteTerms( 
   int *pRC,               /* Result code */
   Fts3Table *p,           /* The FTS table to delete from */
-  sqlite3_value *pRowid,  /* The docid to be deleted */
+  i64 iRowid,             /* The rowid to be deleted */
   u32 *aSz,               /* Sizes of deleted document written here */
   int *pbFound            /* OUT: Set to true if row really does exist */
 ){
@@ -1045,8 +1072,9 @@ static void fts3DeleteTerms(
 
   assert( *pbFound==0 );
   if( *pRC ) return;
-  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
+  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, 0);
   if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pSelect, 1, iRowid);
     if( SQLITE_ROW==sqlite3_step(pSelect) ){
       int i;
       int iLangid = langidFromSelect(p, pSelect);
@@ -2346,7 +2374,7 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){
 ** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
 ** error occurs, an SQLite error code is returned.
 */
-static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
+static int fts3IsEmpty(Fts3Table *p, i64 iRowid, int *pisEmpty){
   sqlite3_stmt *pStmt;
   int rc;
   if( p->zContentTbl ){
@@ -2354,8 +2382,9 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
     *pisEmpty = 0;
     rc = SQLITE_OK;
   }else{
-    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
+    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, 0);
     if( rc==SQLITE_OK ){
+      sqlite3_bind_int64(pStmt, 1, iRowid);
       if( SQLITE_ROW==sqlite3_step(pStmt) ){
         *pisEmpty = sqlite3_column_int(pStmt, 0);
       }
@@ -5197,17 +5226,17 @@ int sqlite3Fts3DeferToken(
 */
 static int fts3DeleteByRowid(
   Fts3Table *p, 
-  sqlite3_value *pRowid, 
+  i64 iRowid,
   int *pnChng,                    /* IN/OUT: Decrement if row is deleted */
   u32 *aSzDel
 ){
   int rc = SQLITE_OK;             /* Return code */
   int bFound = 0;                 /* True if *pRowid really is in the table */
 
-  fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound);
+  fts3DeleteTerms(&rc, p, iRowid, aSzDel, &bFound);
   if( bFound && rc==SQLITE_OK ){
     int isEmpty = 0;              /* Deleting *pRowid leaves the table empty */
-    rc = fts3IsEmpty(p, pRowid, &isEmpty);
+    rc = fts3IsEmpty(p, iRowid, &isEmpty);
     if( rc==SQLITE_OK ){
       if( isEmpty ){
         /* Deleting this row means the whole table is empty. In this case
@@ -5219,10 +5248,10 @@ static int fts3DeleteByRowid(
       }else{
         *pnChng = *pnChng - 1;
         if( p->zContentTbl==0 ){
-          fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
+          fts3SqlExecI64(&rc, p, SQL_DELETE_CONTENT, iRowid);
         }
         if( p->bHasDocsize ){
-          fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
+          fts3SqlExecI64(&rc, p, SQL_DELETE_DOCSIZE, iRowid);
         }
       }
     }
@@ -5231,6 +5260,57 @@ static int fts3DeleteByRowid(
   return rc;
 }
 
+/*
+** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
+** according to the configured languageid_bits= value belonging to
+** FTS table *p.
+**
+** The conversion is as follows:
+**
+**   * The sign bit of iDocid becomes the sign bit of the rowid.
+**
+**   * iLangid is converted to an unsigned integer and stored in
+**     the next most significant Fts3Table.nLanguageidBits bits
+**     of the returned rowid.
+**
+**   * The least signficant (63-nLanguageidBits) of iDocid are
+**     copied to the (63-nLanguageidBits) least signifcant bits of
+**     the returned rowid.
+*/
+i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
+  u64 iRet = iDocid;
+
+  if( p->nLanguageidBits ){
+    int iShift = (63 - p->nLanguageidBits);
+    u64 mask = ((((u64)1 << p->nLanguageidBits) - 1) << iShift);
+
+    iRet &= ~mask;
+    iRet |= (u64)iLangid << iShift;
+  }
+
+  assert( sqlite3Fts3RowidToDocid(p, (i64)iRet)==iDocid );
+  return (i64)iRet;
+}
+
+/*
+** Convert a rowid (iRowid) to a docid according to the languageid_bits=
+** value belonging to FTS table *p.
+*/
+i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){
+  u64 iRet = iRowid;
+  if( p->nLanguageidBits ){
+    static const u64 signbit = ((u64)1 << 63);
+    u64 mask = ((((u64)1 << p->nLanguageidBits)-1) << (63-p->nLanguageidBits));
+
+    if( iRet & signbit ){
+      iRet |= mask;
+    }else{
+      iRet &= ~mask;
+    }
+  }
+  return (i64)iRet;
+}
+
 /*
 ** This function does the work for the xUpdate method of FTS3 virtual
 ** tables. The schema of the virtual table being:
@@ -5242,7 +5322,6 @@ static int fts3DeleteByRowid(
 **       <langid> HIDDEN
 **     );
 **
-** 
 */
 int sqlite3Fts3UpdateMethod(
   sqlite3_vtab *pVtab,            /* FTS3 vtab object */
@@ -5257,6 +5336,7 @@ int sqlite3Fts3UpdateMethod(
   u32 *aSzDel = 0;                /* Sizes of deleted documents */
   int nChng = 0;                  /* Net change in number of documents */
   int bInsertDone = 0;
+  int iLangid = 0;
 
   assert( p->pSegments==0 );
   assert( 
@@ -5276,9 +5356,31 @@ int sqlite3Fts3UpdateMethod(
     goto update_out;
   }
 
-  if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){
-    rc = SQLITE_CONSTRAINT;
-    goto update_out;
+  /* If this is an INSERT or UPDATE, check that the new value for the
+  ** languageid is within range. A languageid can never be a negative 
+  ** value. If the languageid_bits option was specified when this table 
+  ** was created, it must also be less than (2 ^ nLanguageidBits).  
+  **
+  ** Also check that if a non-zero languageid_bits value was configured,
+  ** the specified rowid value must be NULL.
+  */
+  if( nArg>1 ){
+    i64 iLangid64 = sqlite3_value_int64(apVal[2 + p->nColumn + 2]);
+    if( iLangid64<0 
+     || (p->nLanguageidBits && iLangid64>=((i64)1<<p->nLanguageidBits)) 
+    ){
+      rc = SQLITE_CONSTRAINT;
+      goto update_out;
+    }
+    iLangid = (int)iLangid64;
+
+    if( p->nLanguageidBits 
+     && sqlite3_value_type(apVal[0])==SQLITE_NULL
+     && sqlite3_value_type(apVal[1])!=SQLITE_NULL
+    ){
+      rc = SQLITE_CONSTRAINT;
+      goto update_out;
+    }
   }
 
   /* Allocate space to hold the change in document sizes */
@@ -5304,37 +5406,72 @@ int sqlite3Fts3UpdateMethod(
   */
   if( nArg>1 && p->zContentTbl==0 ){
     /* Find the value object that holds the new rowid value. */
-    sqlite3_value *pNewRowid = apVal[3+p->nColumn];
-    if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
-      pNewRowid = apVal[1];
+    sqlite3_value *pNewDocid = apVal[3+p->nColumn];
+    if( sqlite3_value_type(pNewDocid)==SQLITE_NULL ){
+      if( p->nLanguageidBits ){
+        rc = SQLITE_CONSTRAINT;
+        goto update_out;
+      }
+      pNewDocid = apVal[1];
+    }else if( sqlite3_value_type(apVal[0])==SQLITE_NULL 
+           && sqlite3_value_type(apVal[1])!=SQLITE_NULL 
+    ){
+      rc = SQLITE_ERROR;
+      goto update_out;
     }
 
-    if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( 
-        sqlite3_value_type(apVal[0])==SQLITE_NULL
-     || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid)
-    )){
-      /* The new rowid is not NULL (in this case the rowid will be
-      ** automatically assigned and there is no chance of a conflict), and 
-      ** the statement is either an INSERT or an UPDATE that modifies the
-      ** rowid column. So if the conflict mode is REPLACE, then delete any
-      ** existing row with rowid=pNewRowid. 
-      **
-      ** Or, if the conflict mode is not REPLACE, insert the new record into 
-      ** the %_content table. If we hit the duplicate rowid constraint (or any
-      ** other error) while doing so, return immediately.
-      **
-      ** This branch may also run if pNewRowid contains a value that cannot
-      ** be losslessly converted to an integer. In this case, the eventual 
-      ** call to fts3InsertData() (either just below or further on in this
-      ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
-      ** invoked, it will delete zero rows (since no row will have
-      ** docid=$pNewRowid if $pNewRowid is not an integer value).
-      */
-      if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
-        rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel);
-      }else{
-        rc = fts3InsertData(p, apVal, pRowid);
-        bInsertDone = 1;
+    if( sqlite3_value_type(pNewDocid)!=SQLITE_NULL ){
+      int e = sqlite3_value_numeric_type(pNewDocid);
+      i64 iRowid = sqlite3_value_int64(pNewDocid);
+
+      /* Check that the value specified by the user may be losslessly
+      ** converted to an integer. If not, return a "data mismatch" error.  */
+      if( (e!=SQLITE_INTEGER)
+       && (e!=SQLITE_FLOAT || (double)iRowid!=sqlite3_value_double(pNewDocid))
+      ){
+        rc = SQLITE_MISMATCH;
+        goto update_out;
+      }
+
+      if( p->nLanguageidBits ){
+        /* Check for an out-of-range docid value. */
+        if( iRowid>=((i64)1 << (63 - p->nLanguageidBits)) 
+         || iRowid<-1*((i64)1 << (63 - p->nLanguageidBits)) 
+        ){
+          rc = SQLITE_CONSTRAINT;
+          goto update_out;
+        }
+
+        iRowid = sqlite3Fts3DocidToRowid(p, iRowid, iLangid);
+      }
+
+      if( sqlite3_value_type(apVal[0])==SQLITE_NULL
+       || sqlite3_value_int64(apVal[0])!=iRowid
+      ){
+        /* The new rowid is not NULL (in this case the rowid will be
+        ** automatically assigned and there is no chance of a conflict), and 
+        ** the statement is either an INSERT or an UPDATE that modifies the
+        ** rowid column. So if the conflict mode is REPLACE, then delete any
+        ** existing row with rowid=pNewRowid. 
+        **
+        ** Or, if the conflict mode is not REPLACE, insert the new record into 
+        ** the %_content table. If we hit the duplicate rowid constraint (or 
+        ** any other error) while doing so, return immediately.
+        **
+        ** This branch may also run if pNewRowid contains a value that cannot
+        ** be losslessly converted to an integer. In this case, the eventual 
+        ** call to fts3InsertData() (either just below or further on in this
+        ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
+        ** invoked, it will delete zero rows (since no row will have
+        ** docid=$pNewRowid if $pNewRowid is not an integer value).
+        */
+        if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
+          rc = fts3DeleteByRowid(p, iRowid, &nChng, aSzDel);
+        }else{
+          rc = fts3InsertData(p, apVal, 0, iRowid);
+          bInsertDone = 1;
+          *pRowid = iRowid;
+        }
       }
     }
   }
@@ -5345,15 +5482,14 @@ int sqlite3Fts3UpdateMethod(
   /* If this is a DELETE or UPDATE operation, remove the old record. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
-    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
+    rc = fts3DeleteByRowid(p, sqlite3_value_int64(apVal[0]), &nChng, aSzDel);
     isRemove = 1;
   }
   
   /* If this is an INSERT or UPDATE operation, insert the new record. */
   if( nArg>1 && rc==SQLITE_OK ){
-    int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
     if( bInsertDone==0 ){
-      rc = fts3InsertData(p, apVal, pRowid);
+      rc = fts3InsertData(p, apVal, pRowid, 0);
       if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
         rc = FTS_CORRUPT_VTAB;
       }

manifest

@@ -1,5 +1,5 @@
-C Only\sdefault\sHAVE_POSIX_FALLOCATE\son\sfor\slinux,\sand\sthen\sonly\sif\sit\sis\snot\npreviously\sdefined.
-D 2013-06-19T14:49:14.627
+C Add\stests\sfor\smodifying\sthe\sdocid\sand\slanguageid\sfields\sof\san\sfts\stable\swith\sa\snon-zero\slanguageid_bits\sfield.
+D 2013-06-20T19:55:50.400
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 931b3c83abdd1ab3bb389b2130431c2a9ff73b91
+F ext/fts3/fts3.c e88a755aefca0fb98cad59791ea8d02cbeff80f8
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h cb4df04cf886d9920a71df9e8faaa5aae2fa48c6
+F ext/fts3/fts3Int.h 9bef3710aa94fc27b117eca41088aa29ed99d4f1
 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd
 F ext/fts3/fts3_expr.c f8eb1046063ba342c7114eba175cabb31c4a64e7
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9
 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d
-F ext/fts3/fts3_write.c 6a1fc0e922e76b68e594bf7bc33bac72af9dc47b
+F ext/fts3/fts3_write.c af8d9e57097b105659ea621a6504c696d6b3b0ff
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
@@ -546,6 +546,7 @@ F test/fts4aa.test 95f448fb02c4a976968b08d1b4ce134e720946ae
 F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8
 F test/fts4content.test 6efc53b4fd03cab167e6998d2b0b7d4b7d419ee6
 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
+F test/fts4langid2.test 9d75a07687a058f8d0c4036289855ed019e3e128
 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
@@ -706,7 +707,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test 4614301e38398df7fdd5f28f4ed8f272b328251b
-F test/permutations.test d997a947ab8aabb15f763d50a030b3c11e8ef1b6
+F test/permutations.test 5a937ed83e743c260ea9b483c007feb425935bc2
 F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178
 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -1093,7 +1094,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P b9b30d4f9845d212e2d3206abbf2795099e5d71d
-R 051331ae49dcb9481117487fe2ddd3f7
-U drh
-Z f091211d6e3ac84847a4aa6c11062c5e
+P 81527768ef9b39289cfda69d415069b7968379dd
+R 048a66a227c9fbd0111b8d14b4bf151b
+U dan
+Z dc031be52762eed244125900c7455e59

manifest.uuid

@@ -1 +1 @@
-2b2ade92788be623af6f57e37d98994be2cec142
+949425d467947c3905cca3e08750fbaf2ef645db

test/fts4langid2.test

@@ -0,0 +1,332 @@
+# 2012 March 01
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the languageid=xxx FTS4 option.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix fts4langid2
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Test out-of-range values for the languageid_bits= parameter.
+#
+do_catchsql_test 1.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
+} {1 {languageid_bits parameter out of range}}
+
+do_catchsql_test 1.2 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=-1);
+} {1 {languageid_bits parameter out of range}}
+
+do_catchsql_test 1.3 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=0);
+  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=30);
+} {0 {}}
+
+do_execsql_test 1.4 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+}
+
+#-------------------------------------------------------------------------
+# Test out-of-range values in the languageid column.
+#
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=8);
+  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=7);
+}
+
+do_catchsql_test 2.2 {
+  INSERT INTO t1(docid, lid, content) VALUES(1, 256, 'abc def');
+} {1 {constraint failed}}
+
+do_catchsql_test 2.3 {
+  INSERT INTO t2(docid, lid, content) VALUES(1, 128, 'abc def');
+} {1 {constraint failed}}
+
+do_catchsql_test 2.3 {
+  INSERT INTO t1(docid, lid, content) VALUES(1, -1, 'abc def');
+} {1 {constraint failed}}
+
+do_execsql_test 2.4 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+}
+
+#-------------------------------------------------------------------------
+# Test that if languageid_bits is set to a non-zero value it is
+# not possible to specify a non-NULL rowid, even if it is the same
+# as the docid.
+#
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=4);
+  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=0);
+}
+  
+do_catchsql_test 3.2.1 {
+  INSERT INTO t1(rowid, lid, content) VALUES(1, 0, 'abc def');
+} {1 {constraint failed}}
+
+do_catchsql_test 3.2.2 {
+  INSERT INTO t2(rowid, lid, content) VALUES(1, 0, 'abc def');
+} {0 {}}
+
+do_catchsql_test 3.3 {
+  INSERT INTO t1(rowid, docid, lid, content) VALUES(2, 2, 0, 'abc def');
+} {1 {constraint failed}}
+
+do_catchsql_test 3.4 {
+  INSERT INTO t1(lid, content) VALUES(0, 'one two def');
+} {1 {constraint failed}}
+
+do_execsql_test 3.4 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+}
+
+#-------------------------------------------------------------------------
+# Simple tests inserting data with multiple languageid values.
+#
+do_execsql_test 4.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=5);
+}
+
+do_execsql_test 4.2 {
+  INSERT INTO t1 (docid, lid, content) VALUES(1, 0, '1 2 3');
+  INSERT INTO t1 (docid, lid, content) VALUES(1, 1, '1 2 3 4');
+}
+
+do_execsql_test 4.3 {
+  SELECT docid, lid FROM t1;
+} {1 0 1 1}
+
+do_execsql_test 4.4 {
+  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2';
+} {1 0 {1 2 3}}
+
+do_execsql_test 4.5 {
+  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2' AND lid=1;
+} {1 1 {1 2 3 4}}
+
+do_execsql_test 4.6 {
+  UPDATE t1 SET content = 'x y z' || lid;
+  SELECT docid, lid FROM t1;
+} {1 0 1 1}
+
+do_execsql_test 3.4 {
+  DROP TABLE t1;
+}
+
+#-------------------------------------------------------------------------
+# Tests for docid range boundary conditions. 
+#
+for {set bits 1} {$bits <= 30} {incr bits} {
+  do_execsql_test 5.$bits.1 "
+    CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=$bits);
+  "
+
+  set max_docid [expr int(1<<(63-$bits))-1]
+  set min_docid [expr -1*int(1<<(63-$bits))]
+  set max_langid [expr (1<<$bits)-1]
+  set min_langid 0
+
+
+  do_catchsql_test 5.$bits.2.1 {
+    INSERT INTO t1(docid, lid, content) VALUES($max_docid+1, 4, '');
+  } {1 {constraint failed}}
+  do_catchsql_test 5.$bits.2.2 {
+    INSERT INTO t1(docid, lid, content) VALUES($min_docid-1, 4, '');
+  } {1 {constraint failed}}
+
+  do_test 5.$bits.3 {
+    foreach {a b c} "
+      $min_docid  $min_langid  {1 min min x}
+      $min_docid  $max_langid  {2 min max x}
+      $max_docid  $min_langid  {3 max min x}
+      $max_docid  $max_langid  {4 max max x}
+    " {
+      execsql { INSERT INTO t1(docid, lid, content) VALUES($a, $b, $c) }
+    }
+  } {}
+
+  do_execsql_test 5.$bits.4.1 {
+    SELECT docid, lid, content FROM t1 ORDER BY content
+  } "
+      $min_docid  $min_langid  {1 min min x}
+      $min_docid  $max_langid  {2 min max x}
+      $max_docid  $min_langid  {3 max min x}
+      $max_docid  $max_langid  {4 max max x}
+  "
+
+  do_execsql_test 5.$bits.4.2 {
+    SELECT docid, lid, content FROM t1 WHERE lid=$min_langid AND t1 MATCH 'x'
+  } "
+      $min_docid  $min_langid  {1 min min x}
+      $max_docid  $min_langid  {3 max min x}
+  "
+
+  do_execsql_test 5.$bits.4.3 {
+    SELECT docid, lid, content FROM t1 WHERE lid=$max_langid AND t1 MATCH 'x'
+  } "
+      $min_docid  $max_langid  {2 min max x}
+      $max_docid  $max_langid  {4 max max x}
+  "
+
+  do_execsql_test 5.$bits.4.4 {
+    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
+  } "
+      $min_docid  $min_langid  {1 min min x}
+  "
+
+  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
+}
+
+#-------------------------------------------------------------------------
+# Tests for auxilliary functions with langaugeid_bits tables.
+#
+proc mit {blob} {
+  set scan(littleEndian) i*
+  set scan(bigEndian) I*
+  binary scan $blob $scan($::tcl_platform(byteOrder)) r
+  return $r
+}
+db func mit mit
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=4, languageid=lid);
+  INSERT INTO t1(docid,lid,content) VALUES(1, 1, 'one two three four');
+  INSERT INTO t1(docid,lid,content) VALUES(2, 1, 'two three four five');
+  INSERT INTO t1(docid,lid,content) VALUES(3, 1, 'three four five six');
+  INSERT INTO t1(docid,lid,content) VALUES(4, 1, 'four five six seven');
+
+  INSERT INTO t1(docid,lid,content) VALUES(1, 2, 'four three two one');
+  INSERT INTO t1(docid,lid,content) VALUES(2, 2, 'five four three two');
+  INSERT INTO t1(docid,lid,content) VALUES(3, 2, 'six five four three');
+  INSERT INTO t1(docid,lid,content) VALUES(4, 2, 'A B C D');
+}
+
+do_execsql_test 6.2.1 {
+  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=1;
+} {1 {<b>one</b> two three four}}
+do_execsql_test 6.2.2 {
+  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=2;
+} {1 {four three two <b>one</b>}}
+
+do_execsql_test 6.2.1 {
+  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
+} {1 {0 0 4 3} 2 {0 0 0 3}}
+do_execsql_test 6.2.2 {
+  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
+} {1 {0 0 11 3} 2 {0 0 16 3}}
+
+do_execsql_test 6.3.1 {
+  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
+} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
+do_execsql_test 6.3.2 {
+  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
+} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
+do_execsql_test 6.3.3 {
+  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
+} {}
+do_execsql_test 6.3.4 {
+  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
+} {4 {1 1 1 1 1}}
+
+do_execsql_test 6.4 {
+  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
+  INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
+  INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
+  INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
+  INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');
+
+  INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
+  INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
+  INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
+  INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
+}
+
+do_execsql_test 6.4.1 {
+  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
+} {
+  -4 {1 1 1 4 4}
+  -3 {1 1 1 4 4}
+  -2 {1 1 1 4 4}
+  -1 {1 1 1 4 4}
+}
+do_execsql_test 6.4.2 {
+  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
+} {-2 {1 1 4 4 1}}
+
+do_execsql_test 6.5 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+}
+
+#-------------------------------------------------------------------------
+# Tests for querying by docid.
+#
+do_execsql_test 7.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
+  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
+}
+
+do_execsql_test 7.2 {
+  SELECT docid,lid,content FROM t1 WHERE docid=10;
+} {10 10 {abc def}}
+
+do_execsql_test 7.3 {
+  SELECT docid,lid,content FROM t1 WHERE docid<11;
+} {10 10 {abc def}}
+
+do_execsql_test 7.4 {
+  DROP TABLE t1;
+}
+
+#-------------------------------------------------------------------------
+# Tests for sorting by docid.
+#
+do_execsql_test 8.1 {
+  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
+  INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
+  INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
+  INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');
+
+  INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
+  INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
+  INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
+}
+
+do_execsql_test 8.2 {
+  SELECT docid FROM t1 ORDER BY docid;
+} {1 1 2 2 3 3}
+do_execsql_test 8.3 {
+  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
+} {2 3}
+do_execsql_test 8.4 {
+  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
+} {3 2}
+
+# Test that the docid and languageid fields may be updated.
+#
+do_execsql_test 8.5 { UPDATE t1 SET docid=docid+3, lid=0 WHERE lid=5; }
+do_execsql_test 8.6 { SELECT docid FROM t1 ORDER BY docid; } {1 2 3 4 5 6}
+do_execsql_test 8.7 { SELECT docid FROM t1 WHERE t1 MATCH 'A' } {4 6}
+do_execsql_test 8.8 { SELECT docid FROM t1 WHERE t1 MATCH 'A' AND lid=5 } {}
+
+finish_test
+

test/permutations.test

@@ -193,7 +193,8 @@ test_suite "fts3" -prefix "" -description {
   fts3aux1.test fts3comp1.test fts3auto.test
   fts4aa.test fts4content.test
   fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
-  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
+  fts3corrupt2.test fts3first.test fts4langid.test fts4langid2.test
+  fts4merge.test
   fts4check.test fts4unicode.test
 }