Add the fts5 'optimize' command.

FossilOrigin-Name: e749be563d8e738af113bd301770e2f22763ab77

ext/fts5/fts5.c

@@ -1080,6 +1080,8 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
     }
+  }else if( 0==sqlite3_stricmp("optimize", z) ){
+    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
   }else if( 0==sqlite3_stricmp("integrity-check", z) ){
     rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
   }else{

ext/fts5/fts5Int.h

@@ -341,6 +341,7 @@ int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 int sqlite3Fts5IndexReads(Fts5Index *p);
 
 int sqlite3Fts5IndexReinit(Fts5Index *p);
+int sqlite3Fts5IndexOptimize(Fts5Index *p);
 
 /*
 ** End of interface to code in fts5_index.c.
@@ -425,6 +426,7 @@ int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
 
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
+int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 
 /*
 ** End of interface to code in fts5_storage.c.

ext/fts5/fts5_index.c

@@ -41,6 +41,7 @@
 **
 */
 
+#define FTS5_OPT_WORK_UNIT  1000  /* Number of leaf pages per optimize step */
 #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
 #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
 
@@ -3164,6 +3165,12 @@ static void fts5IndexCrisisMerge(
   *ppStruct = pStruct;
 }
 
+static int fts5IndexReturn(Fts5Index *p){
+  int rc = p->rc;
+  p->rc = SQLITE_OK;
+  return rc;
+}
+
 typedef struct Fts5FlushCtx Fts5FlushCtx;
 struct Fts5FlushCtx {
   Fts5Index *pIdx;
@@ -3277,6 +3284,66 @@ static void fts5IndexFlush(Fts5Index *p){
   p->nPendingData = 0;
 }
 
+
+int sqlite3Fts5IndexOptimize(Fts5Index *p){
+  Fts5Config *pConfig = p->pConfig;
+  int i;
+
+  fts5IndexFlush(p);
+  for(i=0; i<=pConfig->nPrefix; i++){
+    Fts5Structure *pStruct = fts5StructureRead(p, i);
+    Fts5Structure *pNew = 0;
+    int nSeg = 0;
+    if( pStruct ){
+      nSeg = fts5StructureCountSegments(pStruct);
+      if( nSeg>1 ){
+        int nByte = sizeof(Fts5Structure);
+        nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
+        pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+      }
+    }
+    if( pNew ){
+      Fts5StructureLevel *pLvl;
+      int nByte = nSeg * sizeof(Fts5StructureSegment);
+      pNew->nLevel = pStruct->nLevel+1;
+      pNew->nWriteCounter = pStruct->nWriteCounter;
+      pLvl = &pNew->aLevel[pStruct->nLevel];
+      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
+      if( pLvl->aSeg ){
+        int iLvl, iSeg;
+        int iSegOut = 0;
+        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+            pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
+            iSegOut++;
+          }
+        }
+        pLvl->nSeg = nSeg;
+      }else{
+        sqlite3_free(pNew);
+        pNew = 0;
+      }
+    }
+
+    if( pNew ){
+      int iLvl = pNew->nLevel-1;
+      while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
+        int nRem = FTS5_OPT_WORK_UNIT;
+        fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem);
+      }
+
+      fts5StructureWrite(p, i, pNew);
+      fts5StructureRelease(pNew);
+    }
+
+    fts5StructureRelease(pStruct);
+  }
+
+  return fts5IndexReturn(p); 
+}
+
+
+
 /*
 ** Return a simple checksum value based on the arguments.
 */
@@ -3788,12 +3855,6 @@ static void fts5SetupPrefixIter(
   sqlite3_free(aBuf);
 }
 
-static int fts5IndexReturn(Fts5Index *p){
-  int rc = p->rc;
-  p->rc = SQLITE_OK;
-  return rc;
-}
-
 /*
 ** Run internal checks to ensure that the FTS index (a) is internally 
 ** consistent and (b) contains entries for which the XOR of the checksums

ext/fts5/fts5_storage.c

@@ -585,6 +585,10 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
   return rc;
 }
 
+int sqlite3Fts5StorageOptimize(Fts5Storage *p){
+  return sqlite3Fts5IndexOptimize(p->pIndex);
+}
+
 /*
 ** Allocate a new rowid. This is used for "external content" tables when
 ** a NULL value is inserted into the rowid column. The new rowid is allocated

ext/fts5/test/fts5optimize.test

@@ -0,0 +1,60 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5optimize
+
+proc rnddoc {nWord} {
+  set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
+  set nVocab [llength $vocab]
+  set ret [list]
+  for {set i 0} {$i < $nWord} {incr i} {
+    lappend ret [lindex $vocab [expr {int(rand() * $nVocab)}]]
+  }
+  return $ret
+}
+
+
+foreach {tn nStep} {
+  1 2
+  2 10
+  3 50
+  4 500
+} {
+if {$tn!=4} continue
+  reset_db
+  db func rnddoc rnddoc
+  do_execsql_test 1.$tn.1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  }
+  do_test 1.$tn.2 {
+    for {set i 0} {$i < $nStep} {incr i} {
+      execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
+    }
+  } {}
+
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  do_execsql_test 1.$tn.4 {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+
+  do_execsql_test 1.$tn.5 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+finish_test
+