On unix, the "PRAGMA fsync_interval=N" command causes an extra fdatasync()

after writing N bytes of content, to force a write-queue flush in the
underlying OS.  This is an experimental hack that is not expected to land
on trunk.

FossilOrigin-Name: b18cc5fee44fb2ab8a48fb3ba92a780090c7ead6af1ebcb67ab1fe214dde709c

ext/zonefile/README.md

@@ -0,0 +1,120 @@
+
+# The Zonefile Extension
+
+## Functionality
+
+### Creating Zonefile Files
+
+To create a new zonefile, first create a database table with the following
+schema:
+
+>     CREATE TABLE data(
+>       k INTEGER PRIMARY KEY,
+>       frame INTEGER DEFAULT -1,   -- frame number.  Automatic if -1
+>       idx INTEGER DEFAULT -1,     -- index of entry within frame.  Auto if -1
+>       v BLOB
+>     );
+
+The table may be created in a persistent or temporary database and may
+take any name, but must contain the columns above. The table must be 
+populated with a row for each key intended to appear in the new zonefile
+file.
+
+Once the table is populated, a zonefile is created using the following
+SQL:
+
+>     SELECT zonefile_write(<file>, <table> [, <parameters>]);
+
+where &lt;file&gt; is the name of the file to create on disk, &lt;table&gt; 
+is the name of the database table to read and optional argument 
+&lt;parameters&gt; is a JSON object containing various attributes that
+influence creation of the zonefile file. 
+
+Currently the only &lt;parameters&gt; attribute supported is 
+<i>maxAutoFrameSize</i> (default value 65536), which sets the maximum 
+uncompressed frame size in bytes for automatically generated zonefile 
+frames.
+
+For example, to create a zonefile named "test.zonefile" based on the
+contents of database table "test_input" and with a maximum automatic
+frame size of 4096 bytes:
+
+>     SELECT zonefile_write('test.zonefile', 'test_input',
+>       '{"maxAutoFrameSize":4096}'
+>     );
+
+### Using (Reading) Zonefile Files
+
+To create a new zonefile table:
+
+>     CREATE VIRTUAL TABLE z1 USING zonefile;
+
+This creates two virtual tables in the database schema. One read-only table
+named "z1", with a schema equivalent to:
+
+>     CREATE TABLE z1(  -- this whole table is read-only
+>       k INTEGER PRIMARY KEY,
+>       v BLOB,
+>       fileid INTEGER,
+>       frame INTEGER,
+>       ofst INTEGER,
+>       sz INTEGER
+>     );
+
+And a read-write table named "z1_files" with a schema like:
+
+>     CREATE TABLE z1_files(
+>       filename TEXT PRIMARY KEY,
+>       ekey BLOB,         -- encryption key
+>       fileid INTEGER,    -- read-only
+>       header JSON HIDDEN -- read-only
+>     );
+
+Both tables are initially empty. To add a zonefile to the index, insert a
+row into the "z1_files" table:
+
+>     INSERT INTO z1_files(filename) VALUES(<filename>);
+
+Currently, any value provided for any column other than "filename" is 
+ignored. Files are removed from the index by deleting rows from the
+z1_files table:
+
+>     DELETE FROM z1_files WHERE filename = <filename>;
+
+Once zonefile files have been added to the index, their contents are 
+visible in table "z1". To retrieve the value associated with a single
+key from one of the zonefile files in the index:
+
+>     SELECT v FROM z1 WHERE k = <key>;
+
+
+## Notes
+
+  *  Contrary to the spec, the implementation uses 32-bit (not 16-bit) frame
+     numbers. So the KeyOffsetPair structure becomes:
+
+     KeyOffsetPair
+     {
+       uint64  key;
+       uint32  frameNo;
+       uint32  frameByteOffset;
+     };
+
+     Also, the ZonefileHeader.numFrames field is now 32-bit. Which makes
+     the ZonefileHeader structure 26 bytes in size. The implementation
+     pads this out to 32 bytes so that the ZoneFileIndex is 8-byte aligned.
+
+  *  Multi-byte integer values are big-endian.
+
+  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are
+     relative to the offset in ZoneFileHeader.byteOffsetFrames. This is
+     necessary as we may not know the offset of the start of the frame data
+     until after the ZoneFileIndex structure is compressed.
+
+  *  Currently there is no support at all for encryption or compression.
+
+  *  Zonefile currently uses json1 to parse the json argument to
+     zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1
+     or otherwise json1-enabled SQLite.
+
+

ext/zonefile/zonefile1.test

@@ -0,0 +1,152 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+set testprefix zonefile1
+load_static_extension db zonefile
+
+do_execsql_test 1.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
+}
+
+do_execsql_test 1.1 {
+  SELECT zonefile_write('test.zonefile', 'zz');
+} {{}}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE z1 USING zonefile;
+  SELECT name FROM sqlite_master WHERE name LIKE 'z1%' ORDER BY 1;
+} {z1 z1_files z1_shadow_file z1_shadow_idx}
+
+do_execsql_test 1.3 {
+  INSERT INTO z1_files(filename) VALUES('test.zonefile');
+  SELECT filename, 
+         json_extract(header, '$.magicNumber'),
+         json_extract(header, '$.numFrames'),
+         json_extract(header, '$.numKeys')
+         FROM z1_files;
+} {test.zonefile 1179332920 1 3}
+
+do_execsql_test 1.4 { SELECT count(*) FROM z1_shadow_idx } 3
+
+do_execsql_test 1.5.1 { SELECT k FROM z1 } {1 2 3}
+do_execsql_test 1.5.2 { SELECT fileid FROM z1 } {1 1 1}
+do_execsql_test 1.5.3 { SELECT frame FROM z1 } {0 0 0}
+do_execsql_test 1.5.4 { SELECT sz FROM z1 } {100 100 100}
+
+do_execsql_test 1.5.5 {
+  SELECT zz.v==z1.v FROM zz, z1 WHERE zz.k=z1.k
+} {1 1 1}
+
+do_execsql_test 1.5 {
+  DELETE FROM z1_files;
+  SELECT * FROM z1_files;
+} {}
+
+do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0
+
+do_execsql_test 1.7 { DROP TABLE z1 }
+
+#-------------------------------------------------------------------------
+
+reset_db
+load_static_extension db zonefile
+
+do_execsql_test 2.0 {
+  CREATE TABLE zz(
+    k INTEGER PRIMARY KEY, 
+    frame INTEGER DEFAULT -1, 
+    idx INTEGER DEFAULT -1, 
+    v BLOB
+  );
+  CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
+  CREATE VIRTUAL TABLE zone USING zonefile;
+}
+
+set nMinByte   0
+set nMaxByte 444
+foreach {zonefile lKey} {
+  test1.zonefile {195 1238 298 405 297}
+  test2.zonefile {124 1624 82 1929}
+  test3.zonefile {932 683 1751 410 41}
+  test4.zonefile {427 1491}
+  test5.zonefile {1004 473 801 394 1672 816 1577}
+  test6.zonefile {1374 1454 1005}
+  test7.zonefile {450 241 319 133}
+  test8.zonefile {1414 900 1406 1917 127 673}
+  test9.zonefile {1192 226 988 1292 718 1345 1675}
+  test10.zonefile {314}
+  test11.zonefile {1177 1597 60 532 291 1164 812}
+  test12.zonefile {1168 1290 1585 939 1916}
+  test13.zonefile {644 1784 1476 1283 433 506}
+  test14.zonefile {1141 1547 1506 364}
+  test15.zonefile {1756 1885 844 1880 1896 354}
+  test16.zonefile {1383 1928 1371}
+  test17.zonefile {93}
+  test18.zonefile {1067}
+  test19.zonefile {642}
+  test20.zonefile {1380 1857}
+  test21.zonefile {288 293 1968 1207 1739 231 300}
+  test22.zonefile {651 1007 607 830 299 1431}
+  test23.zonefile {81 1651 543 1949 256 119 1088}
+  test24.zonefile {1278 2024 682 1115 194 636 1804}
+  test25.zonefile {514 1155 171 2015 791}
+  test26.zonefile {1615 1228 147 1464}
+  test27.zonefile {55 1130 781 678 78}
+  test28.zonefile {1981 1401 1178}
+  test29.zonefile {1754 864 183 1953 1901}
+  test30.zonefile {1461 817}
+  test31.zonefile {1720 1722 686 1833}
+} {
+  forcedelete $zonefile
+  execsql { DELETE FROM zz; }
+  foreach k $lKey {
+    execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
+  }
+  execsql { INSERT INTO rt SELECT k, v FROM zz }
+  breakpoint
+  execsql { 
+    SELECT zonefile_write($zonefile, 'zz', '{"maxAutoFrameSize":2000}');
+    INSERT INTO zone_files(filename) VALUES($zonefile);
+  }
+}
+
+do_execsql_test 2.1 {
+  SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
+}
+do_execsql_test 2.2 {
+  SELECT count(*) FROM zone JOIN rt USING (k);
+} {135}
+do_execsql_test 2.3 {
+  SELECT filename, 
+         json_extract(header, '$.numKeys'),
+         json_extract(header, '$.numFrames')
+  FROM zone_files 
+  WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
+  ORDER BY 1
+} {
+  test19.zonefile 1 1
+  test20.zonefile 2 2
+  test21.zonefile 7 4
+}
+
+
+finish_test
+

main.mk

@@ -372,6 +372,7 @@ TESTSRC += \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
   $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/ext/zonefile/zonefile.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c

manifest

@@ -1,5 +1,5 @@
-C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in.
-D 2018-02-10T02:31:30.872
+C On\sunix,\sthe\s"PRAGMA\sfsync_interval=N"\scommand\scauses\san\sextra\sfdatasync()\nafter\swriting\sN\sbytes\sof\scontent,\sto\sforce\sa\swrite-queue\sflush\sin\sthe\nunderlying\sOS.\s\sThis\sis\san\sexperimental\shack\sthat\sis\snot\sexpected\sto\sland\non\strunk.
+D 2018-02-15T20:00:53.163
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -408,10 +408,13 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
+F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4
+F ext/zonefile/zonefile.c 88d4ba281f47d35ce5989a7c500cfd05e3c8e348426baebcb7bcef314aee08f2
+F ext/zonefile/zonefile1.test 872ec8d549af0f1423601acdfe29390803fab3e14b8d5715f2f4bbd11431f1f5
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk c8c473bd91d553acab3fb0608ddb69fc769c7bcf6d9e258800504bfda86c792b
+F main.mk 0efdf6ac125eb810494d72dd7016ab07ddf3e3fa8615a14aef8491b69818fbe4
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -472,7 +475,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
+F src/os_unix.c 774c789b63aaa4f3e17f5348a336c60bbd94f43e1aeb0a2bc4ffffe60fc178d3
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
@@ -498,7 +501,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 11a2618c227fd13ccad73ee02d1199f9880c59db2b3144fd7432db1980a2577d
-F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828
+F src/test1.c 6b0dfd4f0a1996805bc04381b45df71ead9ed84634d2a47a757ab292febf9026
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1705,7 +1708,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
-R ec6c39c5f311f4ee60e0b56f01a82df2
+P fb1c2277912c55cfae30c18b5434bc193748746395fa7df983cd8a29e5741ff9
+R de6bec149ec0467cfe04c10e1715419d
+T *branch * write-queue-flush-hack
+T *sym-write-queue-flush-hack *
+T -sym-zonefile *
 U drh
-Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7
+Z 7c64ce09377dd8ff73b08359bba5f163

manifest.uuid

@@ -1 +1 @@
-3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
+b18cc5fee44fb2ab8a48fb3ba92a780090c7ead6af1ebcb67ab1fe214dde709c

src/os_unix.c

@@ -209,6 +209,10 @@ struct unixFile {
   unsigned char eFileLock;            /* The type of lock held on this fd */
   unsigned short int ctrlFlags;       /* Behavioral bits.  UNIXFILE_* flags */
   int lastErrno;                      /* The unix errno from last I/O error */
+#ifdef SQLITE_WRITE_QUEUE_FLUSH_HACK
+  unsigned int nUnsyncWrite;          /* Bytes written since last fsync() */
+  unsigned int nUnsyncLimit;          /* Maximum bytes written before fsync() */
+#endif
   void *lockingContext;               /* Locking style specific state */
   UnixUnusedFd *pPreallocatedUnused;  /* Pre-allocated UnixUnusedFd */
   const char *zPath;                  /* Name of the file */
@@ -3368,7 +3372,18 @@ static int unixWrite(
   }
 #endif
  
-  while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))<amt && wrote>0 ){
+  while( 1 ){
+    wrote = seekAndWrite(pFile, offset, pBuf, amt);
+#ifdef SQLITE_WRITE_QUEUE_FLUSH_HACK
+    if( pFile->nUnsyncLimit ){
+      pFile->nUnsyncWrite += wrote;
+      if( pFile->nUnsyncWrite>=pFile->nUnsyncLimit ){
+        fdatasync(pFile->h);
+        pFile->nUnsyncWrite = 0;
+      }
+    }
+#endif
+    if( wrote>=amt || wrote<=0 ) break;
     amt -= wrote;
     offset += wrote;
     pBuf = &((char*)pBuf)[wrote];
@@ -3597,6 +3612,9 @@ static int unixSync(sqlite3_file *id, int flags){
   assert( pFile );
   OSTRACE(("SYNC    %-3d\n", pFile->h));
   rc = full_fsync(pFile->h, isFullsync, isDataOnly);
+#ifdef SQLITE_WRITE_QUEUE_FLUSH_HACK
+  pFile->nUnsyncWrite = 0;
+#endif
   SimulateIOError( rc=1 );
   if( rc ){
     storeLastErrno(pFile, errno);
@@ -3818,6 +3836,17 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
     }
 #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */
 
+#ifdef SQLITE_WRITE_QUEUE_FLUSH_HACK
+    case SQLITE_FCNTL_PRAGMA: {
+      char **azParam = (char**)pArg;
+      if( sqlite3_stricmp(azParam[1],"fsync_interval")==0 ){
+        pFile->nUnsyncLimit = atoi(azParam[2]);
+        return SQLITE_OK;
+      }
+      return SQLITE_NOTFOUND;
+    }
+#endif
+
     case SQLITE_FCNTL_LOCKSTATE: {
       *(int*)pArg = pFile->eFileLock;
       return SQLITE_OK;

src/test1.c

@@ -6992,6 +6992,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
 #ifdef SQLITE_HAVE_ZLIB
   extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
 #endif
+  extern int sqlite3_zonefile_init(sqlite3*,char**,const sqlite3_api_routines*);
   static const struct {
     const char *zExtName;
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
@@ -7016,6 +7017,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
 #ifdef SQLITE_HAVE_ZLIB
     { "zipfile",               sqlite3_zipfile_init              },
 #endif
+    { "zonefile",              sqlite3_zonefile_init             },
   };
   sqlite3 *db;
   const char *zName;