Merge latest trunk changes with this branch.

FossilOrigin-Name: 2e57335081287a2db9fd20d86a8e98218309a37569e1ca1df8deb5fb198d73ff

manifest

@@ -1,5 +1,5 @@
-C Fix\sthe\sCSV\svirtual\stable\sextension\sso\sthat\sit\sworks\swhen\sthe\sdefault\scharacter\nis\sunsigned.
-D 2017-09-18T00:18:31.200
+C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
+D 2017-09-18T08:51:04.787
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -413,7 +413,7 @@ F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 38a225057f5b7d6a2bc403d7a5d3a67f59ee57b73fe7c546221e3c53202a3731
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
-F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
+F src/func.c 8c46246dcb5779e77bf2ef192fa1355460aeec1d3d0b2537608d319bcd7d8778
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -422,7 +422,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 34a58563358fe40979186124d1a3614b9a00c833124d7ebfa8e5d604ed1d2521
+F src/main.c cd03aeca7a9d5f221010d0212b3abdd558e37a317947f8e78dd9162a5d904223
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -441,7 +441,7 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
+F src/os_unix.c dffae52f34dbdc5f270616254b490c61d8e104a7a8fb5ceaa865db25f139ba3c
 F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2a523bf8ec77678b35fe56b43ac24045d2f97ad44d58c6a0894c131feda3eeff
@@ -454,16 +454,16 @@ F src/pragma.c 88d99aa4752894aaf4102eefd09b5e497f4277661aff69f8fff61f8611f19d14
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
-F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
+F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 6ea1cb2f0ab80a5bbaa603c872658c0237d4997d24a455e9b025d74ea08a4615
+F src/select.c 420d3f5d9e24f0d81c28a0d4e5f7d7abf07beae6c25ce8981e51638e9e9ee2ae
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 12aa1f626b3209ffa6a50d9d1e6b4235abd33273a0fcbfeedb66f573a68932b9
+F src/sqliteInt.h cfcfc4238dd5d148bc1802d1588a35e16d7e57abb956779b1c3ffb69b90723ed
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -536,7 +536,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
+F src/wal.c 2a205ec9c9be4e4b2933d32481c4654b4c6b644d8a6c755a5f8c7a10ade64616
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c 21eea981920a13fd3c0ac3d6c128d0a34b22cbec064e4f0603375fe1ffe26ca6
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
-R ccebe3b4b57d7dc8c5482ee2b9f63107
-U drh
-Z a535a0ac2d4006e12331e0812c689362
+P 8b1fc4b9f3e743bffcfbbb8e26991240441778e10512fb502b8eaec460184296 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
+R 4a30c0e0d01ae979fbda091d7fb31409
+U dan
+Z 11c76cfa54aab138b4cbd06a81c15433

manifest.uuid

@@ -1 +1 @@
-42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
+2e57335081287a2db9fd20d86a8e98218309a37569e1ca1df8deb5fb198d73ff

src/func.c

@@ -481,8 +481,9 @@ static void randomFunc(
   sqlite3_value **NotUsed2
 ){
   sqlite_int64 r;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   UNUSED_PARAMETER2(NotUsed, NotUsed2);
-  sqlite3_randomness(sizeof(r), &r);
+  sqlite3FastRandomness(&db->sPrng, sizeof(r), &r);
   if( r<0 ){
     /* We need to prevent a random number of 0x8000000000000000 
     ** (or -9223372036854775808) since when you do abs() of that
@@ -508,6 +509,7 @@ static void randomBlob(
 ){
   int n;
   unsigned char *p;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   assert( argc==1 );
   UNUSED_PARAMETER(argc);
   n = sqlite3_value_int(argv[0]);
@@ -516,7 +518,7 @@ static void randomBlob(
   }
   p = contextMalloc(context, n);
   if( p ){
-    sqlite3_randomness(n, p);
+    sqlite3FastRandomness(&db->sPrng, n, p);
     sqlite3_result_blob(context, (char*)p, n, sqlite3_free);
   }
 }

src/main.c

@@ -2865,6 +2865,7 @@ static int openDatabase(
   db->nDb = 2;
   db->magic = SQLITE_MAGIC_BUSY;
   db->aDb = db->aDbStatic;
+  sqlite3FastPrngInit(&db->sPrng);
 
   assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
   memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));

src/os_unix.c

@@ -46,6 +46,8 @@
 #include "sqliteInt.h"
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
+/* #define SQLITE_SHARED_MAPPING 1 */
+
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
@@ -1115,6 +1117,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -1249,6 +1255,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       closePendingFds(pFile);
       if( pInode->pPrev ){
         assert( pInode->pPrev->pNext==pInode );
@@ -2055,6 +2068,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -3874,6 +3895,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -4774,6 +4798,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -4905,6 +4932,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -5342,6 +5391,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();

src/random.c

@@ -106,6 +106,28 @@ void sqlite3_randomness(int N, void *pBuf){
   sqlite3_mutex_leave(mutex);
 }
 
+/*
+** Initialize a fast PRNG.  A Fast PRNG is called "fast" because it does
+** not need a mutex to operate, though it does use a mutex to initialize.
+** The quality of the randomness is not as good as the global PRNG.
+*/
+void sqlite3FastPrngInit(FastPrng *pPrng){
+  sqlite3_randomness(sizeof(*pPrng), pPrng);
+  pPrng->x |= 1;
+}
+
+/*
+** Generate N bytes of pseudo-randomness using a FastPrng
+*/
+void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){
+  unsigned char *pOut = (unsigned char*)P;
+  while( N-->0 ){
+    pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001);
+    pPrng->y = (pPrng->y)*1103515245 + 12345;
+    *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff;
+  }
+}
+
 #ifndef SQLITE_UNTESTABLE
 /*
 ** For testing purposes, we sometimes want to preserve the state of

src/select.c

@@ -1744,7 +1744,7 @@ int sqlite3ColumnsFromExprList(
         if( zName[j]==':' ) nName = j;
       }
       zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
-      if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
+      if( cnt>3 ) sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt);
     }
     pCol->zName = zName;
     sqlite3ColumnPropertiesFromName(0, pCol);

src/sqliteInt.h

@@ -1065,6 +1065,7 @@ typedef struct NameContext NameContext;
 typedef struct Parse Parse;
 typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
+typedef struct FastPrng FastPrng;
 typedef struct RowSet RowSet;
 typedef struct Savepoint Savepoint;
 typedef struct Select Select;
@@ -1137,6 +1138,14 @@ typedef int VList;
 # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
 #endif
 
+/*
+** State of a simple PRNG used for the per-connection and per-pager
+** pseudo-random number generators.
+*/
+struct FastPrng {
+  unsigned int x, y;
+};
+
 /*
 ** Each database file to be accessed by the system is an instance
 ** of the following structure.  There are normally two of these structures
@@ -1343,6 +1352,7 @@ struct sqlite3 {
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
+  FastPrng sPrng;               /* State of the per-connection PRNG */
   int nChange;                  /* Value returned by sqlite3_changes() */
   int nTotalChange;             /* Value returned by sqlite3_total_changes() */
   int aLimit[SQLITE_N_LIMIT];   /* Limits */
@@ -3812,6 +3822,8 @@ Vdbe *sqlite3GetVdbe(Parse*);
 void sqlite3PrngSaveState(void);
 void sqlite3PrngRestoreState(void);
 #endif
+void sqlite3FastPrngInit(FastPrng*);
+void sqlite3FastRandomness(FastPrng*, int N, void *P);
 void sqlite3RollbackAll(sqlite3*,int);
 void sqlite3CodeVerifySchema(Parse*, int);
 void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb);

src/wal.c

@@ -448,6 +448,7 @@ struct Wal {
   u32 iReCksum;              /* On commit, recalculate checksums from here */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
+  FastPrng sPrng;            /* Random number generator */
 #ifdef SQLITE_DEBUG
   u8 lockError;              /* True if a locking error has occurred */
 #endif
@@ -1323,6 +1324,7 @@ int sqlite3WalOpen(
   pRet->syncHeader = 1;
   pRet->padToSectorBoundary = 1;
   pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
+  sqlite3FastPrngInit(&pRet->sPrng);
 
   /* Open file handle on the write-ahead log file. */
   flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
@@ -1870,7 +1872,7 @@ static int walCheckpoint(
       rc = SQLITE_BUSY;
     }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       assert( pInfo->nBackfill==pWal->hdr.mxFrame );
       rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
@@ -2878,7 +2880,7 @@ static int walRestartLog(Wal *pWal){
     assert( pInfo->nBackfill==pWal->hdr.mxFrame );
     if( pInfo->nBackfill>0 ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
         /* If all readers are using WAL_READ_LOCK(0) (in other words if no
@@ -3094,7 +3096,7 @@ int sqlite3WalFrames(
     sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION);
     sqlite3Put4byte(&aWalHdr[8], szPage);
     sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
-    if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt);
+    if( pWal->nCkpt==0 ) sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt);
     memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
     walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum);
     sqlite3Put4byte(&aWalHdr[24], aCksum[0]);