Merge conflicts

3 files changed, 239 insertions, 171 deletions

diff --git a/lib/libsqlite3/src/os_unix.c b/lib/libsqlite3/src/os_unix.c
index 570bcf3fb88..c40734aac49 100644
--- a/lib/libsqlite3/src/os_unix.c
+++ b/lib/libsqlite3/src/os_unix.c
@@ -84,32 +84,6 @@
 #endif
 
 /*
-** These #defines should enable >2GB file support on Posix if the
-** underlying operating system supports it.  If the OS lacks
-** large file support, these should be no-ops.
-**
-** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch
-** on the compiler command line.  This is necessary if you are compiling
-** on a recent machine (ex: RedHat 7.2) but you want your code to work
-** on an older machine (ex: RedHat 6.0).  If you compile on RedHat 7.2
-** without this option, LFS is enable.  But LFS does not exist in the kernel
-** in RedHat 6.0, so the code won't work.  Hence, for maximum binary
-** portability you should omit LFS.
-**
-** The previous paragraph was written in 2005.  (This paragraph is written
-** on 2008-11-28.) These days, all Linux kernels support large files, so
-** you should probably leave LFS enabled.  But some embedded platforms might
-** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful.
-*/
-#ifndef SQLITE_DISABLE_LFS
-# define _LARGE_FILE       1
-# ifndef _FILE_OFFSET_BITS
-#   define _FILE_OFFSET_BITS 64
-# endif
-# define _LARGEFILE_SOURCE 1
-#endif
-
-/*
 ** standard include files.
 */
 #include <sys/types.h>
@@ -218,11 +192,13 @@ struct unixFile {
   const char *zPath;                  /* Name of the file */
   unixShm *pShm;                      /* Shared memory segment information */
   int szChunk;                        /* Configured by FCNTL_CHUNK_SIZE */
+#if SQLITE_MAX_MMAP_SIZE>0
   int nFetchOut;                      /* Number of outstanding xFetch refs */
   sqlite3_int64 mmapSize;             /* Usable size of mapping at pMapRegion */
   sqlite3_int64 mmapSizeActual;       /* Actual size of mapping at pMapRegion */
   sqlite3_int64 mmapSizeMax;          /* Configured FCNTL_MMAP_SIZE value */
   void *pMapRegion;                   /* Memory mapped region */
+#endif
 #ifdef __QNXNTO__
   int sectorSize;                     /* Device sector size */
   int deviceCharacteristics;          /* Precomputed device characteristics */
@@ -258,6 +234,12 @@ struct unixFile {
 #endif
 };
 
+/* This variable holds the process id (pid) from when the xRandomness()
+** method was called.  If xOpen() is called from a different process id,
+** indicating that a fork() has occurred, the PRNG will be reset.
+*/
+static int randomnessPid = 0;
+
 /*
 ** Allowed values for the unixFile.ctrlFlags bitmask:
 */
@@ -449,6 +431,7 @@ static struct unix_syscall {
   { "fchown",       (sqlite3_syscall_ptr)posixFchown,     0 },
 #define osFchown    ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent)
 
+#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
   { "mmap",       (sqlite3_syscall_ptr)mmap,     0 },
 #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent)
 
@@ -461,6 +444,7 @@ static struct unix_syscall {
   { "mremap",       (sqlite3_syscall_ptr)0,               0 },
 #endif
 #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent)
+#endif
 
 }; /* End of the overrideable system calls */
 
@@ -548,6 +532,15 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){
 }
 
 /*
+** Do not accept any file descriptor less than this value, in order to avoid
+** opening database file using file descriptors that are commonly used for 
+** standard input, output, and error.
+*/
+#ifndef SQLITE_MINIMUM_FILE_DESCRIPTOR
+# define SQLITE_MINIMUM_FILE_DESCRIPTOR 3
+#endif
+
+/*
 ** Invoke open().  Do so multiple times, until it either succeeds or
 ** fails for some reason other than EINTR.
 **
@@ -567,13 +560,23 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){
 static int robust_open(const char *z, int f, mode_t m){
   int fd;
   mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS;
-  do{
+  while(1){
 #if defined(O_CLOEXEC)
     fd = osOpen(z,f|O_CLOEXEC,m2);
 #else
     fd = osOpen(z,f,m2);
 #endif
-  }while( fd<0 && errno==EINTR );
+    if( fd<0 ){
+      if( errno==EINTR ) continue;
+      break;
+    }
+    if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break;
+    osClose(fd);
+    sqlite3_log(SQLITE_WARNING, 
+                "attempt to open \"%s\" as file descriptor %d", z, fd);
+    fd = -1;
+    if( osOpen("/dev/null", f, m)<0 ) break;
+  }
   if( fd>=0 ){
     if( m!=0 ){
       struct stat statbuf;
@@ -1292,6 +1295,15 @@ static int findInodeInfo(
   return SQLITE_OK;
 }
 
+/*
+** Return TRUE if pFile has been renamed or unlinked since it was first opened.
+*/
+static int fileHasMoved(unixFile *pFile){
+  struct stat buf;
+  return pFile->pInode!=0 &&
+         (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino);
+}
+
 
 /*
 ** Check a unixFile that is a database.  Verify the following:
@@ -1326,10 +1338,7 @@ static void verifyDbFile(unixFile *pFile){
     pFile->ctrlFlags |= UNIXFILE_WARNED;
     return;
   }
-  if( pFile->pInode!=0
-   && ((rc = osStat(pFile->zPath, &buf))!=0
-       || buf.st_ino!=pFile->pInode->fileId.ino)
-  ){
+  if( fileHasMoved(pFile) ){
     sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath);
     pFile->ctrlFlags |= UNIXFILE_WARNED;
     return;
@@ -1867,12 +1876,16 @@ end_unlock:
 ** the requested locking level, this routine is a no-op.
 */
 static int unixUnlock(sqlite3_file *id, int eFileLock){
+#if SQLITE_MAX_MMAP_SIZE>0
   assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 );
+#endif
   return posixUnlock(id, eFileLock, 0);
 }
 
+#if SQLITE_MAX_MMAP_SIZE>0
 static int unixMapfile(unixFile *pFd, i64 nByte);
 static void unixUnmapfile(unixFile *pFd);
+#endif
 
 /*
 ** This function performs the parts of the "close file" operation 
@@ -1886,7 +1899,9 @@ static void unixUnmapfile(unixFile *pFd);
 */
 static int closeUnixFile(sqlite3_file *id){
   unixFile *pFile = (unixFile*)id;
+#if SQLITE_MAX_MMAP_SIZE>0
   unixUnmapfile(pFile);
+#endif
   if( pFile->h>=0 ){
     robust_close(pFile, pFile->h, __LINE__);
     pFile->h = -1;
@@ -3091,6 +3106,7 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){
 #endif
   TIMER_START;
   assert( cnt==(cnt&0x1ffff) );
+  assert( id->h>2 );
   cnt &= 0x1ffff;
   do{
 #if defined(USE_PREAD)
@@ -3205,6 +3221,7 @@ static int seekAndWriteFd(
   int rc = 0;                     /* Value returned by system call */
 
   assert( nBuf==(nBuf&0x1ffff) );
+  assert( fd>2 );
   nBuf &= 0x1ffff;
   TIMER_START;
 
@@ -3590,6 +3607,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){
     }
 #endif
 
+#if SQLITE_MAX_MMAP_SIZE>0
     /* If the file was just truncated to a size smaller than the currently
     ** mapped region, reduce the effective mapping size as well. SQLite will
     ** use read() and write() to access data beyond this point from now on.  
@@ -3597,6 +3615,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){
     if( nByte<pFile->mmapSize ){
       pFile->mmapSize = nByte;
     }
+#endif
 
     return SQLITE_OK;
   }
@@ -3686,6 +3705,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){
     }
   }
 
+#if SQLITE_MAX_MMAP_SIZE>0
   if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){
     int rc;
     if( pFile->szChunk<=0 ){
@@ -3698,6 +3718,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){
     rc = unixMapfile(pFile, nByte);
     return rc;
   }
+#endif
 
   return SQLITE_OK;
 }
@@ -3766,6 +3787,11 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       }
       return SQLITE_OK;
     }
+    case SQLITE_FCNTL_HAS_MOVED: {
+      *(int*)pArg = fileHasMoved(pFile);
+      return SQLITE_OK;
+    }
+#if SQLITE_MAX_MMAP_SIZE>0
     case SQLITE_FCNTL_MMAP_SIZE: {
       i64 newLimit = *(i64*)pArg;
       int rc = SQLITE_OK;
@@ -3782,6 +3808,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       }
       return rc;
     }
+#endif
 #ifdef SQLITE_DEBUG
     /* The pager calls this method to signal that it has done
     ** a rollback and that the database is therefore unchanged and
@@ -4044,7 +4071,7 @@ static int unixShmSystemLock(
 #ifdef SQLITE_DEBUG
   { u16 mask;
   OSTRACE(("SHM-LOCK "));
-  mask = (1<<(ofst+n)) - (1<<ofst);
+  mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
   if( rc==SQLITE_OK ){
     if( lockType==F_UNLCK ){
       OSTRACE(("unlock %d ok", ofst));
@@ -4592,22 +4619,20 @@ static int unixShmUnmap(
 # define unixShmUnmap   0
 #endif /* #ifndef SQLITE_OMIT_WAL */
 
+#if SQLITE_MAX_MMAP_SIZE>0
 /*
 ** If it is currently memory mapped, unmap file pFd.
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
-#if SQLITE_MAX_MMAP_SIZE>0
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
     pFd->mmapSize = 0;
     pFd->mmapSizeActual = 0;
   }
-#endif
 }
 
-#if SQLITE_MAX_MMAP_SIZE>0
 /*
 ** Return the system page size.
 */
@@ -4620,9 +4645,7 @@ static int unixGetPagesize(void){
   return (int)sysconf(_SC_PAGESIZE);
 #endif
 }
-#endif /* SQLITE_MAX_MMAP_SIZE>0 */
 
-#if SQLITE_MAX_MMAP_SIZE>0
 /*
 ** Attempt to set the size of the memory mapping maintained by file 
 ** descriptor pFd to nNew bytes. Any existing mapping is discarded.
@@ -4707,7 +4730,6 @@ static void unixRemapfile(
   pFd->pMapRegion = (void *)pNew;
   pFd->mmapSize = pFd->mmapSizeActual = nNew;
 }
-#endif
 
 /*
 ** Memory map or remap the file opened by file-descriptor pFd (if the file
@@ -4726,7 +4748,6 @@ static void unixRemapfile(
 ** code otherwise.
 */
 static int unixMapfile(unixFile *pFd, i64 nByte){
-#if SQLITE_MAX_MMAP_SIZE>0
   i64 nMap = nByte;
   int rc;
 
@@ -4752,10 +4773,10 @@ static int unixMapfile(unixFile *pFd, i64 nByte){
       unixUnmapfile(pFd);
     }
   }
-#endif
 
   return SQLITE_OK;
 }
+#endif /* SQLITE_MAX_MMAP_SIZE>0 */
 
 /*
 ** If possible, return a pointer to a mapping of file fd starting at offset
@@ -4801,6 +4822,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){
 ** may now be invalid and should be unmapped.
 */
 static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){
+#if SQLITE_MAX_MMAP_SIZE>0
   unixFile *pFd = (unixFile *)fd;   /* The underlying database file */
   UNUSED_PARAMETER(iOff);
 
@@ -4819,6 +4841,11 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){
   }
 
   assert( pFd->nFetchOut>=0 );
+#else
+  UNUSED_PARAMETER(fd);
+  UNUSED_PARAMETER(p);
+  UNUSED_PARAMETER(iOff);
+#endif
   return SQLITE_OK;
 }
 
@@ -5150,7 +5177,9 @@ static int fillInUnixFile(
   pNew->pVfs = pVfs;
   pNew->zPath = zFilename;
   pNew->ctrlFlags = (u8)ctrlFlags;
+#if SQLITE_MAX_MMAP_SIZE>0
   pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap;
+#endif
   if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0),
                            "psow", SQLITE_POWERSAFE_OVERWRITE) ){
     pNew->ctrlFlags |= UNIXFILE_PSOW;
@@ -5307,6 +5336,7 @@ static const char *unixTempFileDir(void){
   static const char *azDirs[] = {
      0,
      0,
+     0,
      "/var/tmp",
      "/usr/tmp",
      "/tmp",
@@ -5317,7 +5347,8 @@ static const char *unixTempFileDir(void){
   const char *zDir = 0;
 
   azDirs[0] = sqlite3_temp_directory;
-  if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR");
+  if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR");
+  if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR");
   for(i=0; i<sizeof(azDirs)/sizeof(azDirs[0]); zDir=azDirs[i++]){
     if( zDir==0 ) continue;
     if( osStat(zDir, &buf) ) continue;
@@ -5604,6 +5635,16 @@ static int unixOpen(
        || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL
   );
 
+  /* Detect a pid change and reset the PRNG.  There is a race condition
+  ** here such that two or more threads all trying to open databases at
+  ** the same instant might all reset the PRNG.  But multiple resets
+  ** are harmless.
+  */
+  if( randomnessPid!=getpid() ){
+    randomnessPid = getpid();
+    sqlite3_randomness(0,0);
+  }
+
   memset(p, 0, sizeof(unixFile));
 
   if( eType==SQLITE_OPEN_MAIN_DB ){
@@ -5991,21 +6032,21 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){
   ** tests repeatable.
   */
   memset(zBuf, 0, nBuf);
+  randomnessPid = getpid();  
 #if !defined(SQLITE_TEST)
 # if HAVE_ARC4RANDOM_BUF
   arc4random_buf(zBuf, nBuf);
 # else
   {
-    int pid, fd, got;
+    int fd, got;
     fd = robust_open("/dev/urandom", O_RDONLY, 0);
     if( fd<0 ){
       time_t t;
       time(&t);
       memcpy(zBuf, &t, sizeof(t));
-      pid = getpid();
-      memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid));
-      assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf );
-      nBuf = sizeof(t) + sizeof(pid);
+      memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid));
+      assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf );
+      nBuf = sizeof(t) + sizeof(randomnessPid);
     }else{
       do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR );
       robust_close(0, fd, __LINE__);
diff --git a/lib/libsqlite3/src/parse.y b/lib/libsqlite3/src/parse.y
index e9c8a156351..ad9bedf0b47 100644
--- a/lib/libsqlite3/src/parse.y
+++ b/lib/libsqlite3/src/parse.y
@@ -94,14 +94,6 @@ struct TrigEvent { int a; IdList * b; };
 */
 struct AttachKey { int type;  Token key; };
 
-/*
-** One or more VALUES claues
-*/
-struct ValueList {
-  ExprList *pList;
-  Select *pSelect;
-};
-
 } // end %include
 
 // Input is a single SQL command
@@ -163,13 +155,23 @@ ifnotexists(A) ::= IF NOT EXISTS. {A = 1;}
 temp(A) ::= TEMP.  {A = 1;}
 %endif  SQLITE_OMIT_TEMPDB
 temp(A) ::= .      {A = 0;}
-create_table_args ::= LP columnlist conslist_opt(X) RP(Y). {
-  sqlite3EndTable(pParse,&X,&Y,0);
+create_table_args ::= LP columnlist conslist_opt(X) RP(E) table_options(F). {
+  sqlite3EndTable(pParse,&X,&E,F,0);
 }
 create_table_args ::= AS select(S). {
-  sqlite3EndTable(pParse,0,0,S);
+  sqlite3EndTable(pParse,0,0,0,S);
   sqlite3SelectDelete(pParse->db, S);
 }
+%type table_options {u8}
+table_options(A) ::= .    {A = 0;}
+table_options(A) ::= WITHOUT nm(X). {
+  if( X.n==5 && sqlite3_strnicmp(X.z,"rowid",5)==0 ){
+    A = TF_WithoutRowid;
+  }else{
+    A = 0;
+    sqlite3ErrorMsg(pParse, "unknown table option: %.*s", X.n, X.z);
+  }
+}
 columnlist ::= columnlist COMMA column.
 columnlist ::= column.
 
@@ -192,9 +194,7 @@ columnid(A) ::= nm(X). {
 // An IDENTIFIER can be a generic identifier, or one of several
 // keywords.  Any non-standard keyword can also be an identifier.
 //
-%type id {Token}
-id(A) ::= ID(X).         {A = X;}
-id(A) ::= INDEXED(X).    {A = X;}
+%token_class id  ID|INDEXED.
 
 // The following directive causes tokens ABORT, AFTER, ASC, etc. to
 // fallback to ID if they will not parse as their original value.
@@ -204,8 +204,8 @@ id(A) ::= INDEXED(X).    {A = X;}
   ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
   CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
   IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
-  QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK
-  SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL
+  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
+  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
 %ifdef SQLITE_OMIT_COMPOUND_SELECT
   EXCEPT INTERSECT UNION
 %endif SQLITE_OMIT_COMPOUND_SELECT
@@ -239,8 +239,7 @@ id(A) ::= INDEXED(X).    {A = X;}
 
 // And "ids" is an identifer-or-string.
 //
-%type ids {Token}
-ids(A) ::= ID|STRING(X).   {A = X;}
+%token_class ids  ID|STRING.
 
 // The name of a column or table can be any of the following:
 //
@@ -398,7 +397,7 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). {
 //////////////////////// The SELECT statement /////////////////////////////////
 //
 cmd ::= select(X).  {
-  SelectDest dest = {SRT_Output, 0, 0, 0, 0};
+  SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0};
   sqlite3Select(pParse, X, &dest);
   sqlite3ExplainBegin(pParse->pVdbe);
   sqlite3ExplainSelect(pParse->pVdbe, X);
@@ -408,20 +407,52 @@ cmd ::= select(X).  {
 
 %type select {Select*}
 %destructor select {sqlite3SelectDelete(pParse->db, $$);}
+%type selectnowith {Select*}
+%destructor selectnowith {sqlite3SelectDelete(pParse->db, $$);}
 %type oneselect {Select*}
 %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);}
 
-select(A) ::= oneselect(X).                      {A = X;}
+select(A) ::= with(W) selectnowith(X). {
+  Select *p = X, *pNext, *pLoop;
+  if( p ){
+    int cnt = 0, mxSelect;
+    p->pWith = W;
+    if( p->pPrior ){
+      pNext = 0;
+      for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){
+        pLoop->pNext = pNext;
+        pLoop->selFlags |= SF_Compound;
+      }
+      mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT];
+      if( mxSelect && cnt>mxSelect ){
+        sqlite3ErrorMsg(pParse, "too many terms in compound SELECT");
+      }
+    }
+  }else{
+    sqlite3WithDelete(pParse->db, W);
+  }
+  A = p;
+}
+
+selectnowith(A) ::= oneselect(X).                      {A = X;}
 %ifndef SQLITE_OMIT_COMPOUND_SELECT
-select(A) ::= select(X) multiselect_op(Y) oneselect(Z).  {
-  if( Z ){
-    Z->op = (u8)Y;
-    Z->pPrior = X;
+selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z).  {
+  Select *pRhs = Z;
+  if( pRhs && pRhs->pPrior ){
+    SrcList *pFrom;
+    Token x;
+    x.n = 0;
+    pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0);
+    pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0);
+  }
+  if( pRhs ){
+    pRhs->op = (u8)Y;
+    pRhs->pPrior = X;
     if( Y!=TK_ALL ) pParse->hasCompound = 1;
   }else{
     sqlite3SelectDelete(pParse->db, X);
   }
-  A = Z;
+  A = pRhs;
 }
 %type multiselect_op {int}
 multiselect_op(A) ::= UNION(OP).             {A = @OP;}
@@ -432,6 +463,23 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y)
                  groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). {
   A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset);
 }
+oneselect(A) ::= values(X).    {A = X;}
+
+%type values {Select*}
+%destructor values {sqlite3SelectDelete(pParse->db, $$);}
+values(A) ::= VALUES LP nexprlist(X) RP. {
+  A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0);
+}
+values(A) ::= values(X) COMMA LP exprlist(Y) RP. {
+  Select *pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values,0,0);
+  if( pRight ){
+    pRight->op = TK_ALL;
+    pRight->pPrior = X;
+    A = pRight;
+  }else{
+    A = X;
+  }
+}
 
 // The "distinct" nonterminal is true (1) if the DISTINCT keyword is
 // present and false (0) if it is not.
@@ -573,7 +621,7 @@ indexed_opt(A) ::= NOT INDEXED.      {A.z=0; A.n=1;}
 
 %type using_opt {IdList*}
 %destructor using_opt {sqlite3IdListDelete(pParse->db, $$);}
-using_opt(U) ::= USING LP inscollist(L) RP.  {U = L;}
+using_opt(U) ::= USING LP idlist(L) RP.  {U = L;}
 using_opt(U) ::= .                        {U = 0;}
 
 
@@ -632,15 +680,17 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y).
 /////////////////////////// The DELETE statement /////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
         orderby_opt(O) limit_opt(L). {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE");
   sqlite3DeleteFrom(pParse,X,W);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3DeleteFrom(pParse,X,W);
 }
@@ -655,8 +705,9 @@ where_opt(A) ::= WHERE expr(X).       {A = X.pExpr;}
 ////////////////////////// The UPDATE command ////////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W)
-        orderby_opt(O) limit_opt(L).  {
+cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+        where_opt(W) orderby_opt(O) limit_opt(L).  {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE");
@@ -664,8 +715,9 @@ cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W)
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
         where_opt(W).  {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   sqlite3Update(pParse,X,Y,W,R);
@@ -686,68 +738,30 @@ setlist(A) ::= nm(X) EQ expr(Y). {
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y).
-            {sqlite3Insert(pParse, X, Y.pList, Y.pSelect, F, R);}
-cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S).
-            {sqlite3Insert(pParse, X, 0, S, F, R);}
-cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
-            {sqlite3Insert(pParse, X, 0, 0, F, R);}
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {
+  sqlite3WithPush(pParse, W, 1);
+  sqlite3Insert(pParse, X, S, F, R);
+}
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
+{
+  sqlite3WithPush(pParse, W, 1);
+  sqlite3Insert(pParse, X, 0, F, R);
+}
 
 %type insert_cmd {u8}
 insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
 insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}
 
-// A ValueList is either a single VALUES clause or a comma-separated list
-// of VALUES clauses.  If it is a single VALUES clause then the
-// ValueList.pList field points to the expression list of that clause.
-// If it is a list of VALUES clauses, then those clauses are transformed
-// into a set of SELECT statements without FROM clauses and connected by
-// UNION ALL and the ValueList.pSelect points to the right-most SELECT in
-// that compound.
-%type valuelist {struct ValueList}
-%destructor valuelist {
-  sqlite3ExprListDelete(pParse->db, $$.pList);
-  sqlite3SelectDelete(pParse->db, $$.pSelect);
-}
-valuelist(A) ::= VALUES LP nexprlist(X) RP. {
-  A.pList = X;
-  A.pSelect = 0;
-}
-
-// Since a list of VALUEs is inplemented as a compound SELECT, we have
-// to disable the value list option if compound SELECTs are disabled.
-%ifndef SQLITE_OMIT_COMPOUND_SELECT
-valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. {
-  Select *pRight = sqlite3SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0);
-  if( X.pList ){
-    X.pSelect = sqlite3SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0);
-    X.pList = 0;
-  }
-  A.pList = 0;
-  if( X.pSelect==0 || pRight==0 ){
-    sqlite3SelectDelete(pParse->db, pRight);
-    sqlite3SelectDelete(pParse->db, X.pSelect);
-    A.pSelect = 0;
-  }else{
-    pRight->op = TK_ALL;
-    pRight->pPrior = X.pSelect;
-    pRight->selFlags |= SF_Values;
-    pRight->pPrior->selFlags |= SF_Values;
-    A.pSelect = pRight;
-  }
-}
-%endif SQLITE_OMIT_COMPOUND_SELECT
-
 %type inscollist_opt {IdList*}
 %destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);}
-%type inscollist {IdList*}
-%destructor inscollist {sqlite3IdListDelete(pParse->db, $$);}
+%type idlist {IdList*}
+%destructor idlist {sqlite3IdListDelete(pParse->db, $$);}
 
 inscollist_opt(A) ::= .                       {A = 0;}
-inscollist_opt(A) ::= LP inscollist(X) RP.    {A = X;}
-inscollist(A) ::= inscollist(X) COMMA nm(Y).
+inscollist_opt(A) ::= LP idlist(X) RP.    {A = X;}
+idlist(A) ::= idlist(X) COMMA nm(Y).
     {A = sqlite3IdListAppend(pParse->db,X,&Y);}
-inscollist(A) ::= nm(Y).
+idlist(A) ::= nm(Y).
     {A = sqlite3IdListAppend(pParse->db,0,&Y);}
 
 /////////////////////////// Expression Processing /////////////////////////////
@@ -800,24 +814,24 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). {
 }
 term(A) ::= INTEGER|FLOAT|BLOB(X).  {spanExpr(&A, pParse, @X, &X);}
 term(A) ::= STRING(X).              {spanExpr(&A, pParse, @X, &X);}
-expr(A) ::= REGISTER(X).     {
-  /* When doing a nested parse, one can include terms in an expression
-  ** that look like this:   #1 #2 ...  These terms refer to registers
-  ** in the virtual machine.  #N is the N-th register. */
-  if( pParse->nested==0 ){
-    sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X);
-    A.pExpr = 0;
+expr(A) ::= VARIABLE(X).     {
+  if( X.n>=2 && X.z[0]=='#' && sqlite3Isdigit(X.z[1]) ){
+    /* When doing a nested parse, one can include terms in an expression
+    ** that look like this:   #1 #2 ...  These terms refer to registers
+    ** in the virtual machine.  #N is the N-th register. */
+    if( pParse->nested==0 ){
+      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X);
+      A.pExpr = 0;
+    }else{
+      A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X);
+      if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable);
+    }
   }else{
-    A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X);
-    if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable);
+    spanExpr(&A, pParse, TK_VARIABLE, &X);
+    sqlite3ExprAssignVarNumber(pParse, A.pExpr);
   }
   spanSet(&A, &X, &X);
 }
-expr(A) ::= VARIABLE(X).     {
-  spanExpr(&A, pParse, TK_VARIABLE, &X);
-  sqlite3ExprAssignVarNumber(pParse, A.pExpr);
-  spanSet(&A, &X, &X);
-}
 expr(A) ::= expr(E) COLLATE ids(C). {
   A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C);
   A.zStart = E.zStart;
@@ -829,7 +843,7 @@ expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). {
   spanSet(&A,&X,&Y);
 }
 %endif  SQLITE_OMIT_CAST
-expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). {
+expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). {
   if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){
     sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X);
   }
@@ -839,17 +853,12 @@ expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). {
     A.pExpr->flags |= EP_Distinct;
   }
 }
-expr(A) ::= ID(X) LP STAR RP(E). {
+expr(A) ::= id(X) LP STAR RP(E). {
   A.pExpr = sqlite3ExprFunction(pParse, 0, &X);
   spanSet(&A,&X,&E);
 }
 term(A) ::= CTIME_KW(OP). {
-  /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are
-  ** treated as functions that return constants */
-  A.pExpr = sqlite3ExprFunction(pParse, 0,&OP);
-  if( A.pExpr ){
-    A.pExpr->op = TK_CONST_FUNC;  
-  }
+  A.pExpr = sqlite3ExprFunction(pParse, 0, &OP);
   spanSet(&A, &OP, &OP);
 }
 
@@ -883,10 +892,8 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y).
                                         {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
 expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
 %type likeop {struct LikeOp}
-likeop(A) ::= LIKE_KW(X).     {A.eOperator = X; A.bNot = 0;}
-likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;}
-likeop(A) ::= MATCH(X).       {A.eOperator = X; A.bNot = 0;}
-likeop(A) ::= NOT MATCH(X).   {A.eOperator = X; A.bNot = 1;}
+likeop(A) ::= LIKE_KW|MATCH(X).     {A.eOperator = X; A.bNot = 0;}
+likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;}
 expr(A) ::= expr(X) likeop(OP) expr(Y).  [LIKE_KW]  {
   ExprList *pList;
   pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
@@ -1081,12 +1088,13 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
 
 /* CASE expressions */
 expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). {
-  A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, Z, 0);
+  A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0, 0);
   if( A.pExpr ){
-    A.pExpr->x.pList = Y;
+    A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y;
     sqlite3ExprSetHeight(pParse, A.pExpr);
   }else{
     sqlite3ExprListDelete(pParse->db, Y);
+    sqlite3ExprDelete(pParse->db, Z);
   }
   A.zStart = C.z;
   A.zEnd = &E.z[E.n];
@@ -1193,11 +1201,10 @@ nmnum(A) ::= ON(X).                   {A = X;}
 nmnum(A) ::= DELETE(X).               {A = X;}
 nmnum(A) ::= DEFAULT(X).              {A = X;}
 %endif SQLITE_OMIT_PRAGMA
+%token_class number INTEGER|FLOAT.
 plus_num(A) ::= PLUS number(X).       {A = X;}
 plus_num(A) ::= number(X).            {A = X;}
 minus_num(A) ::= MINUS number(X).     {A = X;}
-number(A) ::= INTEGER|FLOAT(X).       {A = X;}
-
 //////////////////////////// The CREATE TRIGGER command /////////////////////
 
 %ifndef SQLITE_OMIT_TRIGGER
@@ -1226,7 +1233,7 @@ trigger_time(A) ::= .            { A = TK_BEFORE; }
 %destructor trigger_event {sqlite3IdListDelete(pParse->db, $$.b);}
 trigger_event(A) ::= DELETE|INSERT(OP).       {A.a = @OP; A.b = 0;}
 trigger_event(A) ::= UPDATE(OP).              {A.a = @OP; A.b = 0;}
-trigger_event(A) ::= UPDATE OF inscollist(X). {A.a = TK_UPDATE; A.b = X;}
+trigger_event(A) ::= UPDATE OF idlist(X). {A.a = TK_UPDATE; A.b = X;}
 
 foreach_clause ::= .
 foreach_clause ::= FOR EACH ROW.
@@ -1289,12 +1296,8 @@ trigger_cmd(A) ::=
    { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); }
 
 // INSERT
-trigger_cmd(A) ::=
-   insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y).
-   {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);}
-
 trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S).
-               {A = sqlite3TriggerInsertStep(pParse->db, &X, F, 0, S, R);}
+               {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);}
 
 // DELETE
 trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y).
@@ -1400,3 +1403,23 @@ anylist ::= .
 anylist ::= anylist LP anylist RP.
 anylist ::= anylist ANY.
 %endif  SQLITE_OMIT_VIRTUALTABLE
+
+
+//////////////////////// COMMON TABLE EXPRESSIONS ////////////////////////////
+%type with {With*}
+%type wqlist {With*}
+%destructor with {sqlite3WithDelete(pParse->db, $$);}
+%destructor wqlist {sqlite3WithDelete(pParse->db, $$);}
+
+with(A) ::= . {A = 0;}
+%ifndef SQLITE_OMIT_CTE
+with(A) ::= WITH wqlist(W).              { A = W; }
+with(A) ::= WITH RECURSIVE wqlist(W).    { A = W; }
+
+wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+  A = sqlite3WithAdd(pParse, 0, &X, Y, Z);
+}
+wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+  A = sqlite3WithAdd(pParse, W, &X, Y, Z);
+}
+%endif  SQLITE_OMIT_CTE
diff --git a/lib/libsqlite3/src/random.c b/lib/libsqlite3/src/random.c
index ba71245c9eb..145731051bb 100644
--- a/lib/libsqlite3/src/random.c
+++ b/lib/libsqlite3/src/random.c
@@ -57,6 +57,12 @@ void sqlite3_randomness(int N, void *pBuf){
   sqlite3_mutex_enter(mutex);
 #endif
 
+  if( N<=0 ){
+    wsdPrng.isInit = 0;
+    sqlite3_mutex_leave(mutex);
+    return;
+  }
+
   /* Initialize the state of the random number generator once,
   ** the first time this routine is called.  The seed value does
   ** not need to contain a lot of randomness since we are not
@@ -84,7 +90,8 @@ void sqlite3_randomness(int N, void *pBuf){
     wsdPrng.isInit = 1;
   }
 
-  while( N-- ){
+  assert( N>0 );
+  do{
     wsdPrng.i++;
     t = wsdPrng.s[wsdPrng.i];
     wsdPrng.j += t;
@@ -92,7 +99,7 @@ void sqlite3_randomness(int N, void *pBuf){
     wsdPrng.s[wsdPrng.j] = t;
     t += wsdPrng.s[wsdPrng.i];
     *(zBuf++) = wsdPrng.s[t];
-  }
+  }while( --N );
   sqlite3_mutex_leave(mutex);
 }
 
@@ -121,8 +128,5 @@ void sqlite3PrngRestoreState(void){
     sizeof(sqlite3Prng)
   );
 }
-void sqlite3PrngResetState(void){
-  GLOBAL(struct sqlite3PrngType, sqlite3Prng).isInit = 0;
-}
 #endif /* SQLITE_OMIT_BUILTIN_TEST */
 #endif