Handle some boundary cases in memdb associated with OOM faults.

FossilOrigin-Name: b58ca4cb0c921e81efad527c80b220be120263cfdb04528ae26ecf8b8f66f44a

manifest

@@ -1,5 +1,5 @@
-C Simplifications\sto\sthe\smemdb\sVFS.
-D 2018-03-06T19:14:32.801
+C Handle\ssome\sboundary\scases\sin\smemdb\sassociated\swith\sOOM\sfaults.
+D 2018-03-06T20:54:27.966
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1d5a68043cc4d8a6e45b37e2639b148cdd7973aa75e90ec71e12d55cd95e32c0
@@ -425,7 +425,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
-F src/attach.c e79ef463dc0a457281512c5a67c02d886a1a4502001bc9b7eb008e9b7c0f7bf3
+F src/attach.c 6a5619eaf0101b355922cb006e63458312d1067b80499e1df41cdd0ce37af99f
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
@@ -460,7 +460,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
-F src/memdb.c ffdca40946f9cff49c6d0f452648233d1a5234bb1dfa456787ee82333164d126
+F src/memdb.c 3f14ea29aea3a43c39b83c392016a03790692007d751320ea8c19df03bb0b393
 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661
 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53
@@ -1710,7 +1710,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a14fed69d0d4932fc6c71cf8acc5199cca4efbd10bca563a8e86038d6afd5c64
-R b9fff9a384a4ae44f14923e3716823bc
+P 6c3f723a6856fa38ea3f11a36b56f46c5c1fcf17f4daf712e5e0b42562d5f4c6
+R 93b06b41fb0db58473fe3890ab6bb440
 U drh
-Z ed8ab2a467fe746aa3cc4991867879e0
+Z f2d702ac0939e1489f368e13dd72bca2

manifest.uuid

@@ -1 +1 @@
-6c3f723a6856fa38ea3f11a36b56f46c5c1fcf17f4daf712e5e0b42562d5f4c6
+b58ca4cb0c921e81efad527c80b220be120263cfdb04528ae26ecf8b8f66f44a

src/attach.c

@@ -97,7 +97,7 @@ static void attachFunc(
     pVfs = sqlite3_vfs_find("memdb");
     if( pVfs==0 ) return;
     pNew = &db->aDb[db->init.iDb];
-    sqlite3BtreeClose(pNew->pBt);
+    if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt);
     pNew->pBt = 0;
     pNew->pSchema = 0;
     rc = sqlite3BtreeOpen(pVfs, "x", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB);
@@ -232,6 +232,7 @@ static void attachFunc(
     db->init.iDb = 0;
     rc = sqlite3Init(db, &zErrDyn);
     sqlite3BtreeLeaveAll(db);
+    assert( zErrDyn==0 || rc!=SQLITE_OK );
   }
 #ifdef SQLITE_USER_AUTHENTICATION
   if( rc==SQLITE_OK ){
@@ -242,22 +243,24 @@ static void attachFunc(
     }
   }
 #endif
-  if( rc && !REOPEN_AS_MEMDB(db) ){
-    int iDb = db->nDb - 1;
-    assert( iDb>=2 );
-    if( db->aDb[iDb].pBt ){
-      sqlite3BtreeClose(db->aDb[iDb].pBt);
-      db->aDb[iDb].pBt = 0;
-      db->aDb[iDb].pSchema = 0;
-    }
-    sqlite3ResetAllSchemasOfConnection(db);
-    db->nDb = iDb;
-    if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
-      sqlite3OomFault(db);
-      sqlite3DbFree(db, zErrDyn);
-      zErrDyn = sqlite3MPrintf(db, "out of memory");
-    }else if( zErrDyn==0 ){
-      zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile);
+  if( rc ){
+    if( !REOPEN_AS_MEMDB(db) ){
+      int iDb = db->nDb - 1;
+      assert( iDb>=2 );
+      if( db->aDb[iDb].pBt ){
+        sqlite3BtreeClose(db->aDb[iDb].pBt);
+        db->aDb[iDb].pBt = 0;
+        db->aDb[iDb].pSchema = 0;
+      }
+      sqlite3ResetAllSchemasOfConnection(db);
+      db->nDb = iDb;
+      if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
+        sqlite3OomFault(db);
+        sqlite3DbFree(db, zErrDyn);
+        zErrDyn = sqlite3MPrintf(db, "out of memory");
+      }else if( zErrDyn==0 ){
+        zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile);
+      }
     }
     goto attach_error;
   }

src/memdb.c

@@ -472,27 +472,31 @@ unsigned char *sqlite3_serialize(
   rc = zSql ? sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) : SQLITE_NOMEM;
   sqlite3_free(zSql);
   if( rc ) return 0;
-  sqlite3_step(pStmt);
-  sz = sqlite3_column_int64(pStmt, 0)*szPage;
-  if( piSize ) *piSize = sz;
-  if( mFlags & SQLITE_SERIALIZE_NOCOPY ){
+  rc = sqlite3_step(pStmt);
+  if( rc!=SQLITE_ROW ){
     pOut = 0;
   }else{
-    pOut = sqlite3_malloc64( sz );
-    if( pOut ){
-      int nPage = sqlite3_column_int(pStmt, 0);
-      Pager *pPager = sqlite3BtreePager(pBt);
-      int pgno;
-      for(pgno=1; pgno<=nPage; pgno++){
-        DbPage *pPage = 0;
-        unsigned char *pTo = pOut + szPage*(sqlite3_int64)(pgno-1);
-        rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pPage, 0);
-        if( rc==SQLITE_OK ){
-          memcpy(pTo, sqlite3PagerGetData(pPage), szPage);
-        }else{
-          memset(pTo, 0, szPage);
+    sz = sqlite3_column_int64(pStmt, 0)*szPage;
+    if( piSize ) *piSize = sz;
+    if( mFlags & SQLITE_SERIALIZE_NOCOPY ){
+      pOut = 0;
+    }else{
+      pOut = sqlite3_malloc64( sz );
+      if( pOut ){
+        int nPage = sqlite3_column_int(pStmt, 0);
+        Pager *pPager = sqlite3BtreePager(pBt);
+        int pgno;
+        for(pgno=1; pgno<=nPage; pgno++){
+          DbPage *pPage = 0;
+          unsigned char *pTo = pOut + szPage*(sqlite3_int64)(pgno-1);
+          rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pPage, 0);
+          if( rc==SQLITE_OK ){
+            memcpy(pTo, sqlite3PagerGetData(pPage), szPage);
+          }else{
+            memset(pTo, 0, szPage);
+          }
+          sqlite3PagerUnref(pPage);       
         }
-        sqlite3PagerUnref(pPage);       
       }
     }
   }
@@ -536,15 +540,16 @@ int sqlite3_deserialize(
     goto end_deserialize;
   }
   p = memdbFromDbSchema(db, zSchema);
-  /* The memdbFromDbSchema() call can only fail if zSchema is not
-  ** a valid schema name or if the schema is not a memdb schema.  But
-  ** neither of those things can be true here, so failure is not possible */
-  assert( p!=0 );
-  p->aData = pData;
-  p->sz = szDb;
-  p->szMax = szBuf;
-  p->mFlags = mFlags;
-  rc = SQLITE_OK;
+  if( p==0 ){
+    rc = SQLITE_ERROR;
+  }else{
+    p->aData = pData;
+    p->sz = szDb;
+    p->szMax = szBuf;
+    p->mFlags = mFlags;
+    rc = SQLITE_OK;
+  }
+
 end_deserialize:
   sqlite3_finalize(pStmt);
   sqlite3_mutex_leave(db->mutex);