Fix the FTS 'optimize' command on multi-lingual databases.

FossilOrigin-Name: 65fa693729a336e4d905ce72e6b9ccf4faa772bd

ext/fts3/fts3Int.h

@@ -197,7 +197,7 @@ struct Fts3Table {
   /* Precompiled statements used by the implementation. Each of these 
   ** statements is run and reset within a single virtual table API call. 
   */
-  sqlite3_stmt *aStmt[27];
+  sqlite3_stmt *aStmt[28];
 
   char *zReadExprlist;
   char *zWriteExprlist;

ext/fts3/fts3_write.c

@@ -232,6 +232,8 @@ struct SegmentNode {
 
 #define SQL_DELETE_SEGDIR_RANGE       26
 
+#define SQL_SELECT_ALL_LANGID         27
+
 /*
 ** This function is used to obtain an SQLite prepared statement handle
 ** for the statement identified by the second argument. If successful,
@@ -285,6 +287,7 @@ static int fts3SqlStmt(
 /* 25 */  "",
 
 /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
+/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
 
   };
   int rc = SQLITE_OK;
@@ -2229,7 +2232,12 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
 **
 ** Return SQLITE_OK if successful, or an SQLite error code if not.
 */
-static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
+static int fts3SegmentMaxLevel(
+  Fts3Table *p, 
+  int iLangid,
+  int iIndex, 
+  int *pnMax
+){
   sqlite3_stmt *pStmt;
   int rc;
   assert( iIndex>=0 && iIndex<p->nIndex );
@@ -2242,8 +2250,10 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
   */
   rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
   if( rc!=SQLITE_OK ) return rc;
-  sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
-  sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
+  sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+  sqlite3_bind_int(pStmt, 2, 
+      getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
+  );
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
     *pnMax = sqlite3_column_int(pStmt, 0);
   }
@@ -2804,7 +2814,7 @@ static int fts3SegmentMerge(
       rc = SQLITE_DONE;
       goto finished;
     }
-    rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
+    rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
     bIgnoreEmpty = 1;
 
   }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
@@ -3019,16 +3029,29 @@ static void fts3UpdateDocTotals(
 ** iIndex/iLangid combination.
 */
 static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
-  int i;
   int bSeenDone = 0;
-  int rc = SQLITE_OK;
-  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
-    rc = fts3SegmentMerge(p, 0, i, FTS3_SEGCURSOR_ALL);
-    if( rc==SQLITE_DONE ){
-      bSeenDone = 1;
-      rc = SQLITE_OK;
+  int rc;
+  sqlite3_stmt *pAllLangid = 0;
+
+  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+    while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
+      int i;
+      int iLangid = sqlite3_column_int(pAllLangid, 0);
+      for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
+        rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL);
+        if( rc==SQLITE_DONE ){
+          bSeenDone = 1;
+          rc = SQLITE_OK;
+        }
+      }
     }
+    rc2 = sqlite3_reset(pAllLangid);
+    if( rc==SQLITE_OK ) rc = rc2;
   }
+
   sqlite3Fts3SegmentsClose(p);
   sqlite3Fts3PendingTermsClear(p);
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sbug\sin\smerging\sFTS\slanguage\stables\sfor\slanguages\sother\sthan\slanguage\s0.
-D 2012-03-02T11:48:50.564
+C Fix\sthe\sFTS\s'optimize'\scommand\son\smulti-lingual\sdatabases.
+D 2012-03-02T12:26:19.396
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -65,7 +65,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
 F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 8ba2d8ce5db6da67c5e5e7b8a0b90e6d80826546
+F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb
 F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239
 F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
@@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894
 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
-F ext/fts3/fts3_write.c 36fc2e3a28f51ee135a344877c1e4be0a9f45e6e
+F ext/fts3/fts3_write.c 35b98a42f9bbdd28af1b1f3bb0c09ff07090a764
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
-F test/fts4langid.test 7ab7be619d3acb3727e4bef3230ba3dbcf2e0556
+F test/fts4langid.test a793f2da4cbe9c8ad2f49d2a013c6a0ff61e1783
 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P bea257f70f10dd1111d79cabd1e1462dc651704d
-R a3a9247d2c76c9d90f9fc486f3311f0d
+P d281cb8984c911a4c0cce2ec299e1351d8e580e4
+R 2e9c73a6dc49c268fdce6f613b343e49
 U dan
-Z f1e998b56e58f712fe6da1411961b8ef
+Z bf6a4c44b9753bccc8cfbcc7423214ad

manifest.uuid

@@ -1 +1 @@
-d281cb8984c911a4c0cce2ec299e1351d8e580e4
+65fa693729a336e4d905ce72e6b9ccf4faa772bd

test/fts4langid.test

@@ -234,7 +234,7 @@ do_test 2.0 {
   build_multilingual_db_1 db
 } {}
 
-proc do_test_2.1 {tn query res_script} {
+proc do_test_2 {tn query res_script} {
   for {set langid 0} {$langid < 10} {incr langid} {
     rowid_list_set_langid $langid
     set res [eval $res_script]
@@ -242,16 +242,32 @@ proc do_test_2.1 {tn query res_script} {
     set actual [
       execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid}
     ]
-    do_test 2.1.$tn.$langid [list set {} $actual] $res
+    do_test 2.$tn.$langid [list set {} $actual] $res
   }
 }
 
-do_test_2.1 1  {delta}          { rowid_list delta }
-do_test_2.1 2  {"zero one two"} { rowid_list "zero one two" }
-do_test_2.1 3  {zero one two} {
+# Run some queries. 
+do_test_2 1.1  {delta}          { rowid_list delta }
+do_test_2 1.2  {"zero one two"} { rowid_list "zero one two" }
+do_test_2 1.3  {zero one two} {
   and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
 }
-do_test_2.1 4  {"zero one" OR "one two"} {
+do_test_2 1.4  {"zero one" OR "one two"} {
+  or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
+}
+
+# Now try the same tests as above, but after running the 'optimize'
+# command on the FTS table.
+do_execsql_test 2.2 {
+  INSERT INTO t2(t2) VALUES('optimize');
+  SELECT count(*) FROM t2_segdir;
+} {9}
+do_test_2 2.1  {delta}          { rowid_list delta }
+do_test_2 2.2  {"zero one two"} { rowid_list "zero one two" }
+do_test_2 2.3  {zero one two} {
+  and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
+}
+do_test_2 2.4  {"zero one" OR "one two"} {
   or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }