diff options
| author |   landry <landry@openbsd.org> | 2013-06-09 14:42:05 +0000 |
|---|---|---|
| committer | landry <landry@openbsd.org> | 2013-06-09 14:42:05 +0000 |
| commit | cd69e8a5c6f0a2e7f0abf1cec7d4b61d7e815574 (patch) | |
| tree | d06abfde95a07742304802833c937beba3f86c2b /lib/libsqlite3/src | |
| parent | Add test for "true && true && false" and "set -e". (diff) | |
| download | wireguard-openbsd-cd69e8a5c6f0a2e7f0abf1cec7d4b61d7e815574.tar.xz wireguard-openbsd-cd69e8a5c6f0a2e7f0abf1cec7d4b61d7e815574.zip | |
Update to sqlite 3.7.17.
See for changes:
http://www.sqlite.org/releaselog/3_7_16.html
http://www.sqlite.org/releaselog/3_7_16_1.html
http://www.sqlite.org/releaselog/3_7_16_2.html
http://www.sqlite.org/releaselog/3_7_17.html
tested by sebastia@ on vax & sparc, by myself on
hppa/amd64/sparc64/sgi/i386/macppc.
looks ok to espie@
(a lot of kittens died during the preparation of this cvs import)
Diffstat (limited to 'lib/libsqlite3/src')
72 files changed, 3940 insertions, 1396 deletions
diff --git a/lib/libsqlite3/src/analyze.c b/lib/libsqlite3/src/analyze.c index 632fdc1ac14..9a3e9597db0 100644 --- a/lib/libsqlite3/src/analyze.c +++ b/lib/libsqlite3/src/analyze.c @@ -473,7 +473,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -883,7 +883,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; - if( memcmp(z, "unordered", 10)==0 ){ + if( strcmp(z, "unordered")==0 ){ pIndex->bUnordered = 1; break; } diff --git a/lib/libsqlite3/src/attach.c b/lib/libsqlite3/src/attach.c index 6682c914754..b8e12199b57 100644 --- a/lib/libsqlite3/src/attach.c +++ b/lib/libsqlite3/src/attach.c @@ -109,7 +109,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -126,7 +126,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); diff --git a/lib/libsqlite3/src/backup.c b/lib/libsqlite3/src/backup.c index b234716d61d..252f61cfcaa 100644 --- a/lib/libsqlite3/src/backup.c +++ b/lib/libsqlite3/src/backup.c @@ -212,7 +212,12 @@ static int isFatalError(int rc){ ** page iSrcPg from the source database. Copy this data into the ** destination database. */ -static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ Pager * const pDestPager = sqlite3BtreePager(p->pDest); const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); @@ -285,6 +290,9 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ */ memcpy(zOut, zIn, nCopy); ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } } sqlite3PagerUnref(pDestPg); } @@ -389,9 +397,10 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ - rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); } } @@ -454,7 +463,6 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ nDestTruncate = nSrcPage * (pgszSrc/pgszDest); } assert( nDestTruncate>0 ); - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); if( pgszSrc<pgszDest ){ /* If the source page-size is smaller than the destination page-size, @@ -468,6 +476,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ */ const i64 iSize = (i64)pgszSrc * (i64)nSrcPage; sqlite3_file * const pFile = sqlite3PagerFile(pDestPager); + Pgno iPg; + int nDstPage; i64 iOff; i64 iEnd; @@ -478,13 +488,26 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest )); - /* This call ensures that all data required to recreate the original + /* This block ensures that all data required to recreate the original ** database has been stored in the journal for pDestPager and the ** journal synced to disk. So at this point we may safely modify ** the database file in any way, knowing that if a power failure ** occurs, the original database will be reconstructed from the ** journal file. */ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } /* Write the extra pages and truncate the database file as required */ iEnd = MIN(PENDING_BYTE + pgszDest, iSize); @@ -511,6 +534,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerSync(pDestPager); } }else{ + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } @@ -639,7 +663,7 @@ void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ int rc; assert( p->pDestDb ); sqlite3_mutex_enter(p->pDestDb->mutex); - rc = backupOnePage(p, iPage, aData); + rc = backupOnePage(p, iPage, aData, 1); sqlite3_mutex_leave(p->pDestDb->mutex); assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); if( rc!=SQLITE_OK ){ diff --git a/lib/libsqlite3/src/bitvec.c b/lib/libsqlite3/src/bitvec.c index 8d805a6fe5e..52184aa964c 100644 --- a/lib/libsqlite3/src/bitvec.c +++ b/lib/libsqlite3/src/bitvec.c @@ -72,7 +72,7 @@ /* ** A bitmap is an instance of the following structure. ** -** This bitmap records the existance of zero or more bits +** This bitmap records the existence of zero or more bits ** with values between 1 and iSize, inclusive. ** ** There are three possible representations of the bitmap. diff --git a/lib/libsqlite3/src/btree.c b/lib/libsqlite3/src/btree.c index 246843b79e3..3ca60583e3c 100644 --- a/lib/libsqlite3/src/btree.c +++ b/lib/libsqlite3/src/btree.c @@ -43,6 +43,25 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ */ #define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + #ifndef SQLITE_OMIT_SHARED_CACHE /* ** A list of BtShared objects that are eligible for participation @@ -557,6 +576,19 @@ static void btreeClearHasContent(BtShared *pBt){ } /* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; +} + + +/* ** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. ** @@ -595,12 +627,7 @@ static int saveCursorPosition(BtCursor *pCur){ assert( !pCur->apPage[0]->intKey || !pCur->pKey ); if( rc==SQLITE_OK ){ - int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; - } - pCur->iPage = -1; + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } @@ -618,11 +645,15 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && - p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); - if( SQLITE_OK!=rc ){ - return rc; + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>0 ); + btreeReleaseAllCursorPages(p); } } } @@ -1550,13 +1581,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1599,9 +1634,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1609,7 +1645,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -1840,6 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2107,6 +2144,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ } /* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power ** failures. Level 1 is the same as asynchronous (no syncs() occur and @@ -2210,7 +2260,7 @@ int sqlite3BtreeGetPageSize(Btree *p){ ** known that the shared b-tree mutex is held, but the mutex on the ** database handle that owns *p is not. In this case if sqlite3BtreeEnter() ** were to be called, it might collide with some other operation on the -** database handle that owns *p, causing undefined behaviour. +** database handle that owns *p, causing undefined behavior. */ int sqlite3BtreeGetReserveNoMutex(Btree *p){ assert( sqlite3_mutex_held(p->pBt->mutex) ); @@ -2331,7 +2381,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2467,6 +2517,29 @@ page1_init_failed: return rc; } +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the databse. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then @@ -2477,7 +2550,7 @@ page1_init_failed: */ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ assert( pBt->pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); @@ -2595,6 +2668,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ goto trans_begun; } + assert( IfNotOmitAV(pBt->bDoTruncate)==0 ); /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ @@ -2889,7 +2963,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -2911,24 +2985,23 @@ static int relocatePage( static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); /* -** Perform a single step of an incremental-vacuum. If successful, -** return SQLITE_OK. If there is no work to do (and therefore no -** point in calling this function again), return SQLITE_DONE. +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the -** database so that the last page of the file currently in use -** is no longer in use. +** More specificly, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. ** -** If the nFin parameter is non-zero, this function assumes -** that the caller will keep calling incrVacuumStep() until -** it returns SQLITE_DONE or an error, and that nFin is the -** number of pages the database file will contain after this -** process is complete. If nFin is zero, it is assumed that -** incrVacuumStep() will be called a finite amount of times -** which may or may not empty the freelist. A full autovacuum -** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** operation, or false for an incremental vacuum. */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ Pgno nFreeList; /* Number of pages still on the free-list */ int rc; @@ -2953,15 +3026,15 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } if( eType==PTRMAP_FREEPAGE ){ - if( nFin==0 ){ + if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required - ** if nFin is non-zero. In that case, the free-list will be + ** if bCommit is non-zero. In that case, the free-list will be ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -2971,34 +3044,37 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } else { Pgno iFreePg; /* Index of free page to move pLastPg to */ MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } - /* If nFin is zero, this loop runs exactly once and page pLastPg + /* If bCommit is zero, this loop runs exactly once and page pLastPg ** is swapped with the first free page pulled off the free list. ** - ** On the other hand, if nFin is greater than zero, then keep + ** On the other hand, if bCommit is greater than zero, then keep ** looping until a free-page located within the first nFin pages ** of the file is found. */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } do { MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); - }while( nFin!=0 && iFreePg>nFin ); + }while( bCommit && iFreePg>nFin ); assert( iFreePg<iLastPg ); - rc = sqlite3PagerWrite(pLastPg->pDbPage); - if( rc==SQLITE_OK ){ - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); - } + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, bCommit); releasePage(pLastPg); if( rc!=SQLITE_OK ){ return rc; @@ -3006,30 +3082,40 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ } } - if( nFin==0 ){ - iLastPg--; - while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ - if( PTRMAP_ISPAGE(pBt, iLastPg) ){ - MemPage *pPg; - rc = btreeGetPage(pBt, iLastPg, &pPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - } + if( bCommit==0 ){ + do { iLastPg--; - } - sqlite3PagerTruncateImage(pBt->pPager, iLastPg); + }while( iLastPg==PENDING_BYTE_PAGE(pBt) || PTRMAP_ISPAGE(pBt, iLastPg) ); + pBt->bDoTruncate = 1; pBt->nPage = iLastPg; } return SQLITE_OK; } /* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + + return nFin; +} + +/* ** A write-transaction must be opened before calling this function. ** It performs a single unit of work towards an incremental vacuum. ** @@ -3046,11 +3132,24 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( !pBt->autoVacuum ){ rc = SQLITE_DONE; }else{ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[28], pBt->nPage); + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig<nFin ){ + rc = SQLITE_CORRUPT_BKPT; + }else if( nFree>0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; } } sqlite3BtreeLeave(p); @@ -3077,9 +3176,7 @@ static int autoVacuumCommit(BtShared *pBt){ if( !pBt->incrVacuum ){ Pgno nFin; /* Number of pages in database after autovacuuming */ Pgno nFree; /* Number of pages on the freelist initially */ - Pgno nPtrmap; /* Number of PtrMap pages to be freed */ Pgno iFree; /* The next page to be freed */ - int nEntry; /* Number of entries on one ptrmap page */ Pgno nOrig; /* Database size before freeing */ nOrig = btreePagecount(pBt); @@ -3092,26 +3189,20 @@ static int autoVacuumCommit(BtShared *pBt){ } nFree = get4byte(&pBt->pPage1->aData[36]); - nEntry = pBt->usableSize/5; - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } - while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } + nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFin<nOrig ){ + rc = saveAllCursors(pBt, 0, 0); + } for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree); + rc = incrVacuumStep(pBt, nFin, iFree, 1); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[32], 0); put4byte(&pBt->pPage1->aData[36], 0); put4byte(&pBt->pPage1->aData[28], nFin); - sqlite3PagerTruncateImage(pBt->pPager, nFin); + pBt->bDoTruncate = 1; pBt->nPage = nFin; } if( rc!=SQLITE_OK ){ @@ -3119,7 +3210,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } @@ -3166,6 +3257,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ return rc; } } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } #endif rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); sqlite3BtreeLeave(p); @@ -3181,7 +3275,9 @@ static void btreeEndTransaction(Btree *p){ BtShared *pBt = p->pBt; assert( sqlite3BtreeHoldsMutex(p) ); - btreeClearHasContent(pBt); +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ /* If there are other active statements that belong to this database ** handle, downgrade to a read-only transaction. The other statements @@ -3256,6 +3352,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ return rc; } pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3277,27 +3374,6 @@ int sqlite3BtreeCommit(Btree *p){ return rc; } -#ifndef NDEBUG -/* -** Return the number of write-cursors open on this handle. This is for use -** in assert() expressions, so it is only compiled if NDEBUG is not -** defined. -** -** For the purposes of this routine, a write-cursor is any cursor that -** is capable of writing to the databse. That means the cursor was -** originally opened for writing and the cursor has not be disabled -** by having its state changed to CURSOR_FAULT. -*/ -static int countWriteCursors(BtShared *pBt){ - BtCursor *pCur; - int r = 0; - for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; - } - return r; -} -#endif - /* ** This routine sets the state to CURSOR_FAULT and the error ** code to errCode for every cursor on BtShared that pBtree @@ -3369,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3377,8 +3453,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ pBt->nPage = nPage; releasePage(pPage1); } - assert( countWriteCursors(pBt)==0 ); + assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); } btreeEndTransaction(p); @@ -3803,7 +3880,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -4024,7 +4101,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4203,10 +4282,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage<BTCURSOR_MAX_DEPTH ); + assert( pCur->iPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4323,7 +4403,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -4853,21 +4933,23 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** an error. *ppPage and *pPgno are undefined in the event of an error. ** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. ** -** If the "nearby" parameter is not 0, then a (feeble) effort is made to +** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. ** -** If the "exact" parameter is not 0, and the page-number nearby exists -** anywhere on the free-list, then it is guarenteed to be returned. This -** is only used by auto-vacuum databases when allocating a new table. +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. */ static int allocateBtreePage( - BtShared *pBt, - MemPage **ppPage, - Pgno *pPgno, - Pgno nearby, - u8 exact + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ ){ MemPage *pPage1; int rc; @@ -4878,6 +4960,7 @@ static int allocateBtreePage( Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); n = get4byte(&pPage1->aData[36]); @@ -4890,21 +4973,24 @@ static int allocateBtreePage( Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ - /* If the 'exact' parameter was true and a query of the pointer-map + /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } } - *pPgno = nearby; + }else if( eMode==BTALLOC_LE ){ + searchList = 1; } #endif @@ -4917,7 +5003,8 @@ static int allocateBtreePage( /* The code within this loop is run only once if the 'searchList' variable ** is not true. Otherwise, it runs once for each trunk-page on the - ** free-list until the page 'nearby' is located. + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) */ do { pPrevTrunk = pTrunk; @@ -4930,7 +5017,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -4959,11 +5046,13 @@ static int allocateBtreePage( rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList && nearby==iTrunk ){ + }else if( searchList + && (nearby==iTrunk || (iTrunk<nearby && eMode==BTALLOC_LE)) + ){ /* The list is being searched and this trunk page is the page ** to allocate, regardless of whether it has leaves. */ - assert( *pPgno==iTrunk ); + *pPgno = iTrunk; *ppPage = pTrunk; searchList = 0; rc = sqlite3PagerWrite(pTrunk->pDbPage); @@ -4992,7 +5081,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -5026,14 +5115,24 @@ static int allocateBtreePage( unsigned char *aData = pTrunk->aData; if( nearby>0 ){ u32 i; - int dist; closest = 0; - dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); - for(i=1; i<k; i++){ - int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby); - if( d2<dist ){ - closest = i; - dist = d2; + if( eMode==BTALLOC_LE ){ + for(i=0; i<k; i++){ + iPage = get4byte(&aData[8+i*4]); + if( iPage<=nearby ){ + closest = i; + break; + } + } + }else{ + int dist; + dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); + for(i=1; i<k; i++){ + int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby); + if( d2<dist ){ + closest = i; + dist = d2; + } } } }else{ @@ -5047,7 +5146,9 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iPage==mxPage ); - if( !searchList || iPage==nearby ){ + if( !searchList + || (iPage==nearby || (iPage<nearby && eMode==BTALLOC_LE)) + ){ int noContent; *pPgno = iPage; TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" @@ -5060,7 +5161,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5074,8 +5175,26 @@ static int allocateBtreePage( pPrevTrunk = 0; }while( searchList ); }else{ - /* There are no pages on the freelist, so create a new page at the - ** end of the file */ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)); + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; pBt->nPage++; @@ -5090,7 +5209,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5104,7 +5223,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, 1); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5172,7 +5291,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5199,7 +5318,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5245,7 +5364,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -5432,7 +5551,7 @@ static int fillInCell( ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() - ** may misinterpret the uninitialised values and delete the + ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ if( pBt->autoVacuum && rc==SQLITE_OK ){ @@ -6046,7 +6165,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -6905,7 +7024,7 @@ int sqlite3BtreeInsert( insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occured and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey ** variables. @@ -7119,7 +7238,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). */ - rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } @@ -7134,10 +7253,17 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7158,7 +7284,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7234,7 +7360,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; i<pPage->nCell; i++){ pCell = findCell(pPage, i); @@ -7336,7 +7462,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7371,7 +7497,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7381,7 +7507,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7793,7 +7919,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; @@ -8026,7 +8152,7 @@ char *sqlite3BtreeIntegrityCheck( } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist @@ -8265,6 +8391,17 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/lib/libsqlite3/src/btree.h b/lib/libsqlite3/src/btree.h index d4c9fe37d78..ace0f8cd217 100644 --- a/lib/libsqlite3/src/btree.h +++ b/lib/libsqlite3/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); @@ -139,6 +140,7 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an diff --git a/lib/libsqlite3/src/btreeInt.h b/lib/libsqlite3/src/btreeInt.h index b157decec79..ce3c5493f8c 100644 --- a/lib/libsqlite3/src/btreeInt.h +++ b/lib/libsqlite3/src/btreeInt.h @@ -411,6 +411,7 @@ struct BtShared { #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ diff --git a/lib/libsqlite3/src/build.c b/lib/libsqlite3/src/build.c index c21f0172a9b..3c91cdcfdb7 100644 --- a/lib/libsqlite3/src/build.c +++ b/lib/libsqlite3/src/build.c @@ -2100,7 +2100,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ /* Drop all SQLITE_MASTER table and index entries that refer to the ** table. The program name loops through the master table and deletes ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be + ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ @@ -2392,9 +2392,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ -#ifdef SQLITE_OMIT_MERGE_SORT - int regIdxKey; /* Registers containing the index key */ -#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -2422,13 +2419,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ (char *)pKey, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); -#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); -#else - iSorter = iTab; -#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2436,7 +2429,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); -#ifndef SQLITE_OMIT_MERGE_SORT sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); @@ -2447,8 +2439,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + OE_Abort, "indexed columns are not unique", P4_STATIC ); }else{ addr2 = sqlite3VdbeCurrentAddr(v); @@ -2456,30 +2448,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#else - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - addr2 = addr1 + 1; - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); - } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#endif sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); @@ -2594,7 +2562,7 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -2691,10 +2659,8 @@ Index *sqlite3CreateIndex( for(i=0; i<pList->nExpr; i++){ Expr *pExpr = pList->a[i].pExpr; if( pExpr ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( pColl ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } + assert( pExpr->op==TK_COLLATE ); + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -2755,7 +2721,6 @@ Index *sqlite3CreateIndex( const char *zColName = pListItem->zName; Column *pTabCol; int requestedSortOrder; - CollSeq *pColl; /* Collating sequence */ char *zColl; /* Collation sequence name */ for(j=0, pTabCol=pTab->aCol; j<pTab->nCol; j++, pTabCol++){ @@ -2768,11 +2733,10 @@ Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - if( pListItem->pExpr - && (pColl = sqlite3ExprCollSeq(pParse, pListItem->pExpr))!=0 - ){ + if( pListItem->pExpr ){ int nColl; - zColl = pColl->zName; + assert( pListItem->pExpr->op==TK_COLLATE ); + zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -2781,9 +2745,7 @@ Index *sqlite3CreateIndex( nExtra -= nColl; }else{ zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = "BINARY"; - } + if( !zColl ) zColl = "BINARY"; } if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; @@ -2839,7 +2801,7 @@ Index *sqlite3CreateIndex( ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, @@ -3692,12 +3654,19 @@ void sqlite3MayAbort(Parse *pParse){ ** error. The onError parameter determines which (if any) of the statement ** and/or current transaction is rolled back. */ -void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ +void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + int p4type /* P4_STATIC or P4_TRANSIENT */ +){ Vdbe *v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); } /* diff --git a/lib/libsqlite3/src/ctime.c b/lib/libsqlite3/src/ctime.c index 5dee7247478..60595ff88d9 100644 --- a/lib/libsqlite3/src/ctime.c +++ b/lib/libsqlite3/src/ctime.c @@ -48,15 +48,15 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif @@ -147,6 +147,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif @@ -204,11 +207,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -263,9 +261,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_MERGE_SORT - "OMIT_MERGE_SORT", -#endif #ifdef SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif @@ -353,13 +348,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -385,8 +380,11 @@ int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; i<ArraySize(azCompileOpt); i++){ - if( (sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0) - && ( (azCompileOpt[i][n]==0) || (azCompileOpt[i][n]=='=') ) ) return 1; + if( sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0 + && sqlite3CtypeMap[(unsigned char)azCompileOpt[i][n]]==0 + ){ + return 1; + } } return 0; } diff --git a/lib/libsqlite3/src/delete.c b/lib/libsqlite3/src/delete.c index 01a130d65b4..634e115563a 100644 --- a/lib/libsqlite3/src/delete.c +++ b/lib/libsqlite3/src/delete.c @@ -93,30 +93,28 @@ void sqlite3MaterializeView( int iCur /* Cursor number for ephemerial table */ ){ SelectDest dest; - Select *pDup; + Select *pSel; + SrcList *pFrom; sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); - }else{ - sqlite3SelectDelete(db, pDup); - } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); - if( pDup ) pDup->selFlags |= SF_Materialize; + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); } + + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + if( pSel ) pSel->selFlags |= SF_Materialize; + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); } #endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ diff --git a/lib/libsqlite3/src/expr.c b/lib/libsqlite3/src/expr.c index 9ca34ec7b7f..660397e078a 100644 --- a/lib/libsqlite3/src/expr.c +++ b/lib/libsqlite3/src/expr.c @@ -116,12 +116,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ } assert( op!=TK_REGISTER || p->op2!=TK_COLLATE ); if( op==TK_COLLATE ){ - if( db->init.busy ){ - /* Do not report errors when parsing while the schema */ - pColl = sqlite3FindCollSeq(db, ENC(db), p->u.zToken, 0); - }else{ - pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); - } + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); break; } if( p->pTab!=0 @@ -638,7 +633,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ */ ynVar i; for(i=0; i<pParse->nzVar; i++){ - if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ pExpr->iColumn = x = (ynVar)i+1; break; } @@ -1214,6 +1209,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -1456,10 +1452,11 @@ int sqlite3CodeOnce(Parse *pParse){ ** ** The returned value of this function indicates the b-tree type, as follows: ** -** IN_INDEX_ROWID - The cursor was opened on a database table. -** IN_INDEX_INDEX - The cursor was opened on a database index. -** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: @@ -1582,7 +1579,8 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, pKey,P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ @@ -2935,7 +2933,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, + pExpr->affinity, pExpr->u.zToken, 0); } break; @@ -3281,6 +3280,12 @@ void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); sqlite3ExplainPop(pOut); + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } if( i<pList->nExpr-1 ){ sqlite3ExplainNL(pOut); } @@ -3419,8 +3424,8 @@ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } @@ -3533,7 +3538,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ @@ -3653,7 +3658,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: diff --git a/lib/libsqlite3/src/fkey.c b/lib/libsqlite3/src/fkey.c index 2d01e2524b3..ac35bc194cd 100644 --- a/lib/libsqlite3/src/fkey.c +++ b/lib/libsqlite3/src/fkey.c @@ -21,8 +21,9 @@ ** -------------------------- ** ** Foreign keys in SQLite come in two flavours: deferred and immediate. -** If an immediate foreign key constraint is violated, SQLITE_CONSTRAINT -** is returned and the current statement transaction rolled back. If a +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a ** deferred foreign key constraint is violated, no action is taken ** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. @@ -86,7 +87,8 @@ ** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate -** constraint counter is greater than zero, it returns SQLITE_CONSTRAINT +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY ** and the statement transaction is rolled back. An exception is an INSERT ** statement that inserts a single row only (no triggers). In this case, ** instead of using a counter, an exception is thrown immediately if the @@ -142,7 +144,7 @@ ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. ** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. +** search the schema for a unique index on the parent key columns. ** ** If successful, zero is returned. If the parent key is an INTEGER PRIMARY ** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx @@ -178,7 +180,7 @@ ** into pParse. If an OOM error occurs, non-zero is returned and the ** pParse->db->mallocFailed flag is set. */ -static int locateFkeyIndex( +int sqlite3FkLocateIndex( Parse *pParse, /* Parse context to store any error in */ Table *pParent, /* Parent table of FK constraint pFKey */ FKey *pFKey, /* Foreign key to find index for */ @@ -275,7 +277,9 @@ static int locateFkeyIndex( if( !pIdx ){ if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); } sqlite3DbFree(pParse->db, aiCol); return 1; @@ -424,8 +428,8 @@ static void fkLookupParent( ** incrementing a counter. This is necessary as the VM code is being ** generated for will not open a statement transaction. */ assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ @@ -665,8 +669,8 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ ** any modifications to the schema are made. This is because statement ** transactions are not able to rollback schema changes. */ sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, "foreign key constraint failed", P4_STATIC ); if( iSkip ){ @@ -736,7 +740,7 @@ void sqlite3FkCheck( }else{ pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); if( !isIgnoreErrors || db->mallocFailed ) return; if( pTo==0 ){ @@ -816,7 +820,7 @@ void sqlite3FkCheck( continue; } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ if( !isIgnoreErrors || db->mallocFailed ) return; continue; } @@ -871,7 +875,7 @@ u32 sqlite3FkOldmask( } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ for(i=0; i<pIdx->nColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } @@ -997,7 +1001,7 @@ static Trigger *fkActionTrigger( int i; /* Iterator variable */ Expr *pWhen = 0; /* WHEN clause for the trigger */ - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; assert( aiCol || pFKey->nCol==1 ); for(i=0; i<pFKey->nCol; i++){ diff --git a/lib/libsqlite3/src/func.c b/lib/libsqlite3/src/func.c index d4655ad7e96..e40fdad55a8 100644 --- a/lib/libsqlite3/src/func.c +++ b/lib/libsqlite3/src/func.c @@ -694,6 +694,13 @@ static int patternCompare( } /* +** The sqlite3_strglob() interface. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + +/* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing ** only. @@ -963,6 +970,62 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } /* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc( argc*4 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i<argc; i++){ + sqlite3_int64 x; + unsigned c; + x = sqlite3_value_int64(argv[i]); + if( x<0 || x>0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); +} + +/* ** The hex() function. Interpret the argument as a blob. Return ** a hexadecimal rendering as text. */ @@ -1589,6 +1652,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), diff --git a/lib/libsqlite3/src/global.c b/lib/libsqlite3/src/global.c index f5da7e7f1f4..7b02cf2130e 100644 --- a/lib/libsqlite3/src/global.c +++ b/lib/libsqlite3/src/global.c @@ -156,6 +156,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/lib/libsqlite3/src/hash.h b/lib/libsqlite3/src/hash.h index 990a2d6e22d..82b7c58c710 100644 --- a/lib/libsqlite3/src/hash.h +++ b/lib/libsqlite3/src/hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ diff --git a/lib/libsqlite3/src/insert.c b/lib/libsqlite3/src/insert.c index 89a5dc8a13e..9a5661f59a7 100644 --- a/lib/libsqlite3/src/insert.c +++ b/lib/libsqlite3/src/insert.c @@ -1245,7 +1245,7 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); + SQLITE_CONSTRAINT_NOTNULL, onError, regData+i); zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); @@ -1285,7 +1285,8 @@ void sqlite3GenerateConstraintChecks( }else{ zConsName = 0; } - sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zConsName, P4_DYNAMIC); } sqlite3VdbeResolveLabel(v, allOk); } @@ -1316,8 +1317,8 @@ void sqlite3GenerateConstraintChecks( case OE_Rollback: case OE_Abort: case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); break; } case OE_Replace: { @@ -1444,7 +1445,8 @@ void sqlite3GenerateConstraintChecks( sqlite3StrAccumAppend(&errMsg, pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, 0); sqlite3DbFree(errMsg.db, zErr); break; } @@ -1852,8 +1854,8 @@ static int xferOptimization( if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_PRIMARYKEY, + onError, "PRIMARY KEY must be unique", P4_STATIC); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); }else if( pDest->pIndex==0 ){ diff --git a/lib/libsqlite3/src/journal.c b/lib/libsqlite3/src/journal.c index 06605cc9566..fed27be3e38 100644 --- a/lib/libsqlite3/src/journal.c +++ b/lib/libsqlite3/src/journal.c @@ -59,6 +59,14 @@ static int createFile(JournalFile *p){ assert(p->iSize<=p->nBuf); rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + } } } return rc; diff --git a/lib/libsqlite3/src/legacy.c b/lib/libsqlite3/src/legacy.c index ebab2de37d9..94649ae705a 100644 --- a/lib/libsqlite3/src/legacy.c +++ b/lib/libsqlite3/src/legacy.c @@ -38,7 +38,6 @@ int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -46,12 +45,12 @@ int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -108,11 +107,8 @@ int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } diff --git a/lib/libsqlite3/src/main.c b/lib/libsqlite3/src/main.c index b52d4744431..39f60421e68 100644 --- a/lib/libsqlite3/src/main.c +++ b/lib/libsqlite3/src/main.c @@ -496,6 +496,19 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -885,10 +898,16 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* If we reach this point, it means that the database connection has ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** pased to sqlite3_close (meaning that it is a zombie). Therefore, + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + /* Free any outstanding Savepoint structures. */ sqlite3CloseSavepoints(db); @@ -989,6 +1008,15 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + for(i=0; i<db->nDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ @@ -1002,10 +1030,11 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( db->flags&SQLITE_InternChanges ){ + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; @@ -1017,6 +1046,110 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ } /* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* ** Return a static string that describes the kind of error specified in the ** argument. */ @@ -2316,6 +2449,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/lib/libsqlite3/src/memjournal.c b/lib/libsqlite3/src/memjournal.c index 3e66e215b2a..05725948f65 100644 --- a/lib/libsqlite3/src/memjournal.c +++ b/lib/libsqlite3/src/memjournal.c @@ -230,7 +230,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* diff --git a/lib/libsqlite3/src/os.c b/lib/libsqlite3/src/os.c index b5e918a7272..be2ea4cfc01 100644 --- a/lib/libsqlite3/src/os.c +++ b/lib/libsqlite3/src/os.c @@ -141,6 +141,26 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + /* ** The next group of routines are convenience wrappers around the ** VFS methods. diff --git a/lib/libsqlite3/src/os.h b/lib/libsqlite3/src/os.h index 1ec7d4ba11c..070a2ddd17f 100644 --- a/lib/libsqlite3/src/os.h +++ b/lib/libsqlite3/src/os.h @@ -99,14 +99,6 @@ # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ @@ -259,6 +251,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/lib/libsqlite3/src/os_unix.c b/lib/libsqlite3/src/os_unix.c index 315f1501888..abc23a452e7 100644 --- a/lib/libsqlite3/src/os_unix.c +++ b/lib/libsqlite3/src/os_unix.c @@ -126,7 +126,7 @@ #include <time.h> #include <sys/time.h> #include <errno.h> -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include <sys/mman.h> #endif @@ -225,6 +225,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -249,7 +254,9 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + #endif + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -273,6 +280,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -307,6 +315,17 @@ struct unixFile { #endif /* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + +/* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). ** The difference is important when using a pointer to the function. @@ -337,7 +356,7 @@ static int openDirectory(const char*, int*); ** to all overrideable system calls. */ static struct unix_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -412,11 +431,7 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[13].pCurrent) -#if SQLITE_ENABLE_LOCKING_STYLE { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, -#else - { "fchmod", (sqlite3_syscall_ptr)0, 0 }, -#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -441,8 +456,18 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) - { "umask", (sqlite3_syscall_ptr)umask, 0 }, -#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) }; /* End of the overrideable system calls */ @@ -548,14 +573,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ */ static int robust_open(const char *z, int f, mode_t m){ int fd; - mode_t m2; - mode_t origM = 0; - if( m==0 ){ - m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; - }else{ - m2 = m; - origM = osUmask(0); - } + mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS; do{ #if defined(O_CLOEXEC) fd = osOpen(z,f|O_CLOEXEC,m2); @@ -563,12 +581,20 @@ static int robust_open(const char *z, int f, mode_t m){ fd = osOpen(z,f,m2); #endif }while( fd<0 && errno==EINTR ); - if( m ){ - osUmask(origM); - } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } #if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) - if( fd>=0 ) osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif + } return fd; } @@ -774,7 +800,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -1110,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -1277,6 +1301,50 @@ static int findInodeInfo( /* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + +/* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut ** to a non-zero value otherwise *pResOut is set to zero. The return value @@ -1806,9 +1874,13 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1821,6 +1893,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -1847,6 +1920,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -1915,7 +1989,7 @@ static int nolockClose(sqlite3_file *id) { /****************************************************************************** ************************* Begin dot-file Locking ****************************** ** -** The dotfile locking implementation uses the existance of separate lock +** The dotfile locking implementation uses the existence of separate lock ** files (really a directory) to control access to the database. This works ** on just about every filesystem imaginable. But there are serious downsides: ** @@ -1930,7 +2004,7 @@ static int nolockClose(sqlite3_file *id) { ** ** Dotfile locking works by creating a subdirectory in the same directory as ** the database and with the same name but with a ".lock" extension added. -** The existance of a lock directory implies an EXCLUSIVE lock. All other +** The existence of a lock directory implies an EXCLUSIVE lock. All other ** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. */ @@ -3078,6 +3152,8 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=0 ); + assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3088,6 +3164,23 @@ static int unixRead( ); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3103,46 +3196,59 @@ static int unixRead( } /* -** Seek to the offset in id->offset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. -** -** To avoid stomping the errno value on a failed write the lastErrno value -** is set before returning. +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). */ -static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - assert( cnt==(cnt&0x1ffff) ); - cnt &= 0x1ffff; +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; TIMER_START; + #if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); #elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); #else do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); return -1; } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); #endif + TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -3192,6 +3298,23 @@ static int unixWrite( } #endif +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3419,7 +3542,7 @@ static int unixSync(sqlite3_file *id, int flags){ } /* Also fsync the directory containing the file if the DIRSYNC flag - ** is set. This is a one-time occurrance. Many systems (examples: AIX) + ** is set. This is a one-time occurrence. Many systems (examples: AIX) ** are unable to fsync a directory, so ignore errors on the fsync. */ if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ @@ -3474,6 +3597,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nByte<pFile->mmapSize ){ + pFile->mmapSize = nByte; + } + return SQLITE_OK; } } @@ -3562,6 +3693,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3629,6 +3773,18 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ){ + pFile->mmapSizeMax = newLimit; + if( newLimit<pFile->mmapSize ) pFile->mmapSize = newLimit; + } + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3941,7 +4097,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4181,24 +4337,32 @@ static int unixShmMap( if( sStat.st_size<nByte ){ /* The requested memory region does not exist. If bExtend is set to ** false, exit early. *pp will be set to NULL and SQLITE_OK returned. - ** - ** Alternatively, if bExtend is true, use ftruncate() to allocate - ** the requested memory region. */ - if( !bExtend ) goto shmpage_out; -#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE - if( osFallocate(pShmNode->h, sStat.st_size, nByte)!=0 ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } -#else - if( robust_ftruncate(pShmNode->h, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); - goto shmpage_out; + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } } -#endif } } @@ -4214,7 +4378,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4432,6 +4596,236 @@ static int unixShmUnmap( #endif /* #ifndef SQLITE_OMIT_WAL */ /* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} +#endif + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ +#if SQLITE_MAX_MMAP_SIZE>0 + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } +#endif + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + +/* ** Here ends the implementation of all sqlite3_file methods. ** ********************** End sqlite3_file Methods ******************************* @@ -4489,7 +4883,9 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4506,7 +4902,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ @@ -4757,11 +5153,12 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; } - if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ + if( strcmp(pVfs->zName,"unix-excl")==0 ){ pNew->ctrlFlags |= UNIXFILE_EXCL; } @@ -4793,7 +5190,7 @@ static int fillInUnixFile( unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ - /* If an error occured in findInodeInfo(), close the file descriptor + /* If an error occurred in findInodeInfo(), close the file descriptor ** immediately, before releasing the mutex. findInodeInfo() may fail ** in two scenarios: ** @@ -4892,15 +5289,15 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } @@ -5431,7 +5828,7 @@ static int unixDelete( } /* -** Test the existance of or access permissions of file zPath. The +** Test the existence of or access permissions of file zPath. The ** test performed depends on the value of flags: ** ** SQLITE_ACCESS_EXISTS: Return 1 if the file exists @@ -6994,7 +7391,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/lib/libsqlite3/src/os_win.c b/lib/libsqlite3/src/os_win.c index 107370c41be..aeb08814b3d 100644 --- a/lib/libsqlite3/src/os_win.c +++ b/lib/libsqlite3/src/os_win.c @@ -150,11 +150,20 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -308,7 +317,7 @@ static int sqlite3_os_type = 0; ** to all overrideable system calls. */ static struct win_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -988,7 +997,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** (if available). */ -void sqlite3_win32_write_debug(char *zBuf, int nBuf){ +void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ @@ -1514,7 +1523,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ } #endif if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); @@ -1557,7 +1566,7 @@ static int winLogErrorAtLine( for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, - "os_win.c:%d: (%d) %s(%s) - %s", + "os_win.c:%d: (%lu) %s(%s) - %s", iLine, lastErrno, zFunc, zPath, zMsg ); @@ -1621,9 +1630,10 @@ static void logIoerr(int nRetry){ /************************************************************************* ** This section contains code for WinCE only. */ +#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** Windows CE does not have a localtime() function. So create a -** substitute. +** The MSVC CRT on Windows CE may not have a localtime() function. So +** create a substitute. */ #include <time.h> struct tm *__cdecl localtime(const time_t *t) @@ -1647,6 +1657,7 @@ struct tm *__cdecl localtime(const time_t *t) y.tm_sec = pTm.wSecond; return &y; } +#endif #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] @@ -1668,15 +1679,17 @@ static void winceMutexAcquire(HANDLE h){ ** Create the mutex and shared memory used for locking in the file ** descriptor pFile */ -static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ +static int winceCreateLock(const char *zFilename, winFile *pFile){ LPWSTR zTok; LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; BOOL bInit = TRUE; zName = utf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return FALSE; + return SQLITE_IOERR_NOMEM; } /* Initialize the local lockdata */ @@ -1693,9 +1706,10 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, "winceCreateLock1", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); sqlite3_free(zName); - return FALSE; + return SQLITE_IOERR; } /* Acquire the mutex before continuing */ @@ -1712,41 +1726,49 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ - if (osGetLastError() == ERROR_ALREADY_EXISTS){ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ bInit = FALSE; } sqlite3_free(zName); /* If we succeeded in making the shared memory handle, map it. */ - if (pFile->hShared){ + if( pFile->hShared ){ pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ - if (!pFile->shared){ + if( !pFile->shared ){ pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_ERROR, pFile->lastErrno, - "winceCreateLock2", zFilename); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; osCloseHandle(pFile->hShared); pFile->hShared = NULL; } } /* If shared memory could not be created, then close the mutex and fail */ - if (pFile->hShared == NULL){ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } winceMutexRelease(pFile->hMutex); osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; - return FALSE; + return SQLITE_IOERR; } /* Initialize the shared memory if we're supposed to */ - if (bInit) { + if( bInit ){ memset(pFile->shared, 0, sizeof(winceLock)); } winceMutexRelease(pFile->hMutex); - return TRUE; + return SQLITE_OK; } /* @@ -1825,7 +1847,8 @@ static BOOL winceLockFile( } /* Want a pending lock? */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ /* If no pending lock has been acquired, then acquire it */ if (pFile->shared->bPending == 0) { pFile->shared->bPending = TRUE; @@ -1835,7 +1858,8 @@ static BOOL winceLockFile( } /* Want a reserved lock? */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToLockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ if (pFile->shared->bReserved == 0) { pFile->shared->bReserved = TRUE; pFile->local.bReserved = TRUE; @@ -1878,7 +1902,8 @@ static BOOL winceUnlockFile( /* Did we just have a reader lock? */ else if (pFile->local.nReaders){ - assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE || nNumberOfBytesToUnlockLow == 1); + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); pFile->local.nReaders --; if (pFile->local.nReaders == 0) { @@ -1889,7 +1914,8 @@ static BOOL winceUnlockFile( } /* Releasing a pending lock */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bPending){ pFile->local.bPending = FALSE; pFile->shared->bPending = FALSE; @@ -1897,7 +1923,8 @@ static BOOL winceUnlockFile( } } /* Releasing a reserved lock */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToUnlockLow == 1){ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ if (pFile->local.bReserved) { pFile->local.bReserved = FALSE; pFile->shared->bReserved = FALSE; @@ -2000,6 +2027,8 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ DWORD dwRet; /* Value returned by SetFilePointer() */ DWORD lastErrno; /* Value returned by GetLastError() */ + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -2007,7 +2036,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occured, it is also necessary to call + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -2017,9 +2046,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #else /* @@ -2036,13 +2067,20 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #endif } +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS methods */ +static int winUnmapfile(winFile*); +#endif + /* ** Close a file. ** @@ -2062,7 +2100,14 @@ static int winClose(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); #endif - OSTRACE(("CLOSE %d\n", pFile->h)); + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + rc = winUnmapfile(pFile); + if( rc!=SQLITE_OK ) return rc; +#endif + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2082,11 +2127,11 @@ static int winClose(sqlite3_file *id){ sqlite3_free(pFile->zDeleteOnClose); } #endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); if( rc ){ pFile->h = NULL; } OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -2111,11 +2156,33 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -2129,6 +2196,7 @@ static int winRead( DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } @@ -2136,9 +2204,11 @@ static int winRead( if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); return SQLITE_IOERR_SHORT_READ; } + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2152,7 +2222,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc = 0; /* True if error has occured, else false */ + int rc = 0; /* True if error has occurred, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -2161,7 +2231,26 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -2214,13 +2303,16 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2230,11 +2322,12 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); - - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -2248,14 +2341,25 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nByte<pFile->mmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } @@ -2295,13 +2399,14 @@ static int winSync(sqlite3_file *id, int flags){ || (flags&0x0F)==SQLITE_SYNC_FULL ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); #else @@ -2320,9 +2425,11 @@ static int winSync(sqlite3_file *id, int flags){ rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, "winSync", pFile->zPath); } @@ -2337,7 +2444,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ int rc = SQLITE_OK; assert( id!=0 ); + assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + #if SQLITE_OS_WINRT { FILE_STANDARD_INFO info; @@ -2366,6 +2476,8 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ } } #endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; } @@ -2407,6 +2519,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ */ static int getReadLock(winFile *pFile){ int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ #if SQLITE_OS_WINCE /* @@ -2432,6 +2545,7 @@ static int getReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } + OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2441,6 +2555,7 @@ static int getReadLock(winFile *pFile){ static int unlockReadLock(winFile *pFile){ int res; DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } @@ -2454,6 +2569,7 @@ static int unlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "unlockReadLock", pFile->zPath); } + OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2492,14 +2608,15 @@ static int winLock(sqlite3_file *id, int locktype){ DWORD lastErrno = NO_ERROR; assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as ** sqlite3OsEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2527,7 +2644,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(res))); if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -2572,14 +2690,12 @@ static int winLock(sqlite3_file *id, int locktype){ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -2597,12 +2713,14 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2616,20 +2734,23 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); } *pResOut = rc; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); return SQLITE_OK; } @@ -2650,8 +2771,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -2672,6 +2793,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2699,17 +2822,21 @@ static int getTempname(int nBuf, char *zBuf); */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -2724,20 +2851,25 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); } } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("win32"); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -2752,17 +2884,32 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ }else{ a[1] = win32IoerrRetryDelay; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { - char *zTFile = sqlite3_malloc( pFile->pVfs->mxPathname ); + char *zTFile = sqlite3MallocZero( pFile->pVfs->mxPathname ); if( zTFile ){ getTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } +#endif } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; } @@ -2790,8 +2937,6 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -#ifndef SQLITE_OMIT_WAL - /* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. @@ -2800,6 +2945,8 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ SYSTEM_INFO winSysInfo; +#ifndef SQLITE_OMIT_WAL + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -2923,6 +3070,9 @@ static int winShmSystemLock( /* Access to the winShmNode object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); @@ -2940,11 +3090,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -2964,6 +3112,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -2971,15 +3121,13 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; i<p->nRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } - if( p->hFile.h != INVALID_HANDLE_VALUE ){ + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); winClose((sqlite3_file *)&p->hFile); SimulateIOErrorBenign(0); @@ -3059,7 +3207,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); if( SQLITE_OK!=rc ){ goto shm_open_err; @@ -3256,9 +3404,9 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); return rc; } @@ -3379,8 +3527,8 @@ static int winShmMap( NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; @@ -3394,8 +3542,8 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ @@ -3433,6 +3581,230 @@ shmpage_out: #endif /* #ifndef SQLITE_OMIT_WAL */ /* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapSizeActual = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + +/* ** Here ends the implementation of all sqlite3_file methods. ** ********************** End sqlite3_file Methods ******************************* @@ -3443,7 +3815,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3459,7 +3831,9 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -3523,6 +3897,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3536,6 +3911,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3548,6 +3924,7 @@ static int getTempname(int nBuf, char *zBuf){ nTempPath = sqlite3Strlen30(zTempPath); if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } @@ -3565,8 +3942,8 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = 0; zBuf[j+1] = 0; - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; } /* @@ -3635,9 +4012,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3648,6 +4023,9 @@ static int winOpen( )); #endif + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + /* Check the following statements are true: ** ** (a) Exactly one of the READWRITE and READONLY flags must be set, and @@ -3674,8 +4052,9 @@ static int winOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); - assert( id!=0 ); - UNUSED_PARAMETER(pVfs); + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; #if SQLITE_OS_WINRT if( !sqlite3_temp_directory ){ @@ -3684,15 +4063,15 @@ static int winOpen( } #endif - pFile->h = INVALID_HANDLE_VALUE; - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ if( !zUtf8Name ){ assert(isDelete && !isOpenJournal); + memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -3708,11 +4087,13 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -3803,9 +4184,8 @@ static int winOpen( #endif logIoerr(cnt); - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = lastErrno; @@ -3813,7 +4193,9 @@ static int winOpen( sqlite3_free(zConverted); if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } @@ -3827,26 +4209,18 @@ static int winOpen( } } - memset(pFile, 0, sizeof(*pFile)); - pFile->pMethod = &winIoMethod; - pFile->h = h; - pFile->lastErrno = NO_ERROR; - pFile->pVfs = pVfs; -#ifndef SQLITE_OMIT_WAL - pFile->pShm = 0; -#endif - pFile->zPath = zName; - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ - pFile->ctrlFlags |= WINFILE_PSOW; - } + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && !winceCreateLock(zName, pFile) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK ){ osCloseHandle(h); sqlite3_free(zConverted); - return SQLITE_CANTOPEN_BKPT; + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -3856,6 +4230,25 @@ static int winOpen( sqlite3_free(zConverted); } + pFile->pMethod = &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + OpenCounter(+1); return rc; } @@ -3886,6 +4279,8 @@ static int winDelete( UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; @@ -3900,7 +4295,8 @@ static int winDelete( attr = sAttrData.dwFileAttributes; }else{ lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ }else{ rc = SQLITE_ERROR; @@ -3912,7 +4308,8 @@ static int winDelete( #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ }else{ rc = SQLITE_ERROR; @@ -3939,7 +4336,8 @@ static int winDelete( attr = osGetFileAttributesA(zConverted); if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ }else{ rc = SQLITE_ERROR; @@ -3968,12 +4366,12 @@ static int winDelete( logIoerr(cnt); } sqlite3_free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); return rc; } /* -** Check the existance and status of a file. +** Check the existence and status of a file. */ static int winAccess( sqlite3_vfs *pVfs, /* Not used on win32 */ @@ -3988,8 +4386,12 @@ static int winAccess( UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -4040,6 +4442,8 @@ static int winAccess( assert(!"Invalid flags argument"); } *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); return SQLITE_OK; } @@ -4107,16 +4511,12 @@ static int winFullPathname( */ char zOut[MAX_PATH+1]; memset(zOut, 0, MAX_PATH+1); - cygwin_conv_to_win32_path(zRelative, zOut); /* POSIX to Win32 */ + cygwin_conv_path(CCP_POSIX_TO_WIN_A|CCP_RELATIVE, zRelative, zOut, + MAX_PATH+1); sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", sqlite3_data_directory, zOut); }else{ - /* - ** NOTE: The Cygwin docs state that the maximum length needed - ** for the buffer passed to cygwin_conv_to_full_win32_path - ** is MAX_PATH. - */ - cygwin_conv_to_full_win32_path(zRelative, zFull); + cygwin_conv_path(CCP_POSIX_TO_WIN_A, zRelative, zFull, nFull); } return SQLITE_OK; #endif @@ -4274,9 +4674,9 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); getLastErrorMsg(osGetLastError(), nBuf, zBufOut); } -static void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ UNUSED_PARAMETER(pVfs); - return (void(*)(void))osGetProcAddressA((HANDLE)pHandle, zSymbol); + return (void(*)(void))osGetProcAddressA((HANDLE)pH, zSym); } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); @@ -4374,7 +4774,8 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #endif /* 2^32 - to avoid use of LL and warnings in gcc */ static const sqlite3_int64 max32BitValue = - (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; #if SQLITE_OS_WINCE SYSTEMTIME time; @@ -4483,7 +4884,6 @@ int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4491,8 +4891,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; diff --git a/lib/libsqlite3/src/pager.c b/lib/libsqlite3/src/pager.c index 5879cf760af..1c6a84fea4b 100644 --- a/lib/libsqlite3/src/pager.c +++ b/lib/libsqlite3/src/pager.c @@ -273,7 +273,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. -** If no error occured, all that remains is to finalize the journal to +** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** @@ -521,7 +521,7 @@ struct PagerSavepoint { ** ** doNotSpill, doNotSyncSpill ** -** These two boolean variables control the behaviour of cache-spills +** These two boolean variables control the behavior of cache-spills ** (calls made by the pcache module to the pagerStress() routine to ** write cached data to the file-system in order to free up memory). ** @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -766,6 +771,16 @@ static const unsigned char aJournalMagic[] = { #endif /* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + +/* ** The maximum legal page number is (2^31 - 1). */ #define PAGER_MAX_PGNO 2147483647 @@ -1399,7 +1414,7 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initialiser */ + /* The random check-hash initializer */ sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); /* The initial database size */ @@ -1838,6 +1853,8 @@ static int pager_error(Pager *pPager, int rc){ return rc; } +static int pager_truncate(Pager *pPager, Pgno nPage); + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1891,7 +1908,7 @@ static int pager_error(Pager *pPager, int rc){ ** to the first error encountered (the journal finalization one) is ** returned. */ -static int pager_end_transaction(Pager *pPager, int hasMaster){ +static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ @@ -1977,7 +1994,17 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); } + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ @@ -2016,7 +2043,7 @@ static void pagerUnlockAndRollback(Pager *pPager){ sqlite3EndBenignMalloc(); }else if( !pPager->exclusiveMode ){ assert( pPager->eState==PAGER_READER ); - pager_end_transaction(pPager, 0); + pager_end_transaction(pPager, 0, 0); } } pager_unlock(pPager); @@ -2240,7 +2267,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2631,6 +2658,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -2731,7 +2759,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -2791,7 +2821,7 @@ end_playback: rc = sqlite3PagerSync(pPager); } if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && zMaster[0] && res ){ @@ -2801,6 +2831,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size @@ -2822,11 +2856,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2838,11 +2871,13 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_WAL + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -2921,12 +2956,17 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3070,6 +3110,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3332,6 +3373,29 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + +/* ** Free as much memory as possible from the pager. */ void sqlite3PagerShrink(Pager *pPager){ @@ -3566,6 +3630,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3719,7 +3784,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** dirty page were to be discarded from the cache via the pagerStress() ** routine, pagerStress() would not write the current page content to ** the database file. If a savepoint transaction were rolled back after -** this happened, the correct behaviour would be to restore the current +** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either ** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the @@ -3743,12 +3808,26 @@ static void assertTruncateConstraint(Pager *pPager){ ** function does not actually modify the database file on disk. It ** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSize>=nPage ); assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; - assertTruncateConstraint(pPager); + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ } @@ -3778,6 +3857,81 @@ static int pagerSyncHotJournal(Pager *pPager){ } /* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + +/* ** Shutdown the page cache. Free all memory and close all files. ** ** If a transaction was in progress when this routine is called, that @@ -3797,6 +3951,7 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4058,7 +4213,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; @@ -4612,6 +4769,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; @@ -4801,6 +4959,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ goto failed; } if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4898,9 +5061,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || USEFETCH(pPager) + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4926,7 +5091,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ @@ -4935,6 +5100,16 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } } } @@ -4976,7 +5151,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -5035,13 +5210,27 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5052,6 +5241,39 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5110,9 +5332,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -5165,7 +5391,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5500,6 +5730,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -5699,6 +5930,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ @@ -5885,36 +6121,6 @@ int sqlite3PagerCommitPhaseOne( #endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - */ - if( pPager->dbSize<pPager->dbOrigSize - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - } - pPager->dbSize = dbSize; - } - /* Write the master journal name into the journal file. If a master ** journal file name has already been written to the journal file, ** or if zMaster is NULL (no master journal), then this call is a no-op. @@ -5942,11 +6148,14 @@ int sqlite3PagerCommitPhaseOne( goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); @@ -6019,7 +6228,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -6064,11 +6273,11 @@ int sqlite3PagerRollback(Pager *pPager){ if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster); + rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); if( rc==SQLITE_OK ) rc = rc2; }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ int eState = pPager->eState; - rc = pager_end_transaction(pPager, 0); + rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. @@ -6083,7 +6292,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager @@ -6466,7 +6675,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); } @@ -6816,11 +7026,12 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -6911,6 +7122,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/lib/libsqlite3/src/pager.h b/lib/libsqlite3/src/pager.h index 90f8e6af787..6f659136e27 100644 --- a/lib/libsqlite3/src/pager.h +++ b/lib/libsqlite3/src/pager.h @@ -79,6 +79,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + +/* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for ** a detailed description of each routine. @@ -102,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/lib/libsqlite3/src/pcache.h b/lib/libsqlite3/src/pcache.h index b9135fd8593..f4d4ad71c10 100644 --- a/lib/libsqlite3/src/pcache.h +++ b/lib/libsqlite3/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/lib/libsqlite3/src/pragma.c b/lib/libsqlite3/src/pragma.c index d3061b234f9..3056a7d8e2b 100644 --- a/lib/libsqlite3/src/pragma.c +++ b/lib/libsqlite3/src/pragma.c @@ -184,6 +184,9 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ { "sql_trace", SQLITE_SqlTrace }, { "vdbe_listing", SQLITE_VdbeListing }, { "vdbe_trace", SQLITE_VdbeTrace }, + { "vdbe_addoptrace", SQLITE_VdbeAddopTrace}, + { "vdbe_debug", SQLITE_SqlTrace | SQLITE_VdbeListing + | SQLITE_VdbeTrace }, #endif #ifndef SQLITE_OMIT_CHECK { "ignore_check_constraints", SQLITE_IgnoreChecks }, @@ -316,7 +319,7 @@ void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -399,11 +402,12 @@ void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; @@ -742,6 +746,43 @@ void sqlite3Pragma( }else /* + ** PRAGMA [database.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; +#endif + returnSingleInt(pParse, "mmap_size", sz); + } + }else + + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" ** @@ -948,11 +989,14 @@ void sqlite3Pragma( if( sqlite3ReadSchema(pParse) ) goto pragma_out; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - int i; + int i, k; int nHidden = 0; Column *pCol; + Index *pPk; + for(pPk=pTab->pIndex; pPk && pPk->autoIndex!=2; pPk=pPk->pNext){} sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); @@ -975,8 +1019,14 @@ void sqlite3Pragma( }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp2(v, OP_Integer, - (pCol->colFlags&COLFLAG_PRIMKEY)!=0, 6); + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; ALWAYS(k<=pTab->nCol) && pPk->aiColumn[k-1]!=i; k++){} + } + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -992,6 +1042,7 @@ void sqlite3Pragma( pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); @@ -1018,6 +1069,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); @@ -1081,6 +1133,7 @@ void sqlite3Pragma( int i = 0; sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); @@ -1114,6 +1167,122 @@ void sqlite3Pragma( }else #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + if( sqlite3StrICmp(zLeft, "foreign_key_check")==0 ){ + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + sqlite3CodeVerifySchema(pParse, iDb); + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || pTab->pFKey==0 ) continue; + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + if( pParent==0 ) break; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + } + }else{ + k = 0; + break; + } + } + if( pFK ) break; + if( pParse->nTab<i ) pParse->nTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); + for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3LocateTable(pParse, 0, pFK->zTo, zDb); + assert( pParent!=0 ); + pIdx = 0; + aiCols = 0; + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 ); + addrOk = sqlite3VdbeMakeLabel(v); + if( pIdx==0 ){ + int iKey = pFK->aCol[0].iFrom; + assert( iKey>=0 && iKey<pTab->nCol ); + if( iKey!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); + sqlite3ColumnDefault(v, pTab, iKey, regRow); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); + } + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else{ + for(j=0; j<pFK->nCol; j++){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, + aiCols ? aiCols[j] : pFK->aCol[0].iFrom, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regRow, pFK->nCol, regKey); + sqlite3VdbeChangeP4(v, -1, + sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + } + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); + sqlite3VdbeJumpHere(v, addrTop); + } + }else +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ @@ -1398,6 +1567,11 @@ void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = <integer> ** + ** PRAGMA [database.]freelist_count = <integer> + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = <integer> + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -1419,10 +1593,14 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; @@ -1611,7 +1789,7 @@ void sqlite3Pragma( }else #endif #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ + if( sqlite3StrICmp(zLeft, "activate_extensions")==0 && zRight ){ #ifdef SQLITE_HAS_CODEC if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ sqlite3_activate_see(&zRight[4]); diff --git a/lib/libsqlite3/src/prepare.c b/lib/libsqlite3/src/prepare.c index 5ac8de7296f..d78d83cbd87 100644 --- a/lib/libsqlite3/src/prepare.c +++ b/lib/libsqlite3/src/prepare.c @@ -179,7 +179,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -259,11 +259,15 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ if( iDb==0 ){ +#ifndef SQLITE_OMIT_UTF16 u8 encoding; /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; ENC(db) = encoding; +#else + ENC(db) = SQLITE_UTF8; +#endif }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ @@ -400,7 +404,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ @@ -423,7 +427,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ int sqlite3ReadSchema(Parse *pParse){ @@ -650,7 +654,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); diff --git a/lib/libsqlite3/src/resolve.c b/lib/libsqlite3/src/resolve.c index 51aab7792b8..91efcaa1a16 100644 --- a/lib/libsqlite3/src/resolve.c +++ b/lib/libsqlite3/src/resolve.c @@ -150,6 +150,35 @@ static int nameInUsingClause(IdList *pUsing, const char *zCol){ return 0; } +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +int sqlite3MatchSpanName( + const char *zSpan, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up @@ -206,6 +235,20 @@ static int lookupName( pExpr->pTab = 0; ExprSetIrreducible(pExpr); + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + for(i=0; i<db->nDb; i++){ + assert( db->aDb[i].zName ); + if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + } + /* Start at the inner-most context and move outward until a match is found */ while( pNC && cnt==0 ){ ExprList *pEList; @@ -214,31 +257,36 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; i<pSrcList->nSrc; i++, pItem++){ Table *pTab; - int iDb; Column *pCol; pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( pTab->nCol>0 ); - if( zTab ){ - if( pItem->zAlias ){ - char *zTabName = pItem->zAlias; - if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - }else{ - char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; - } - if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ - continue; + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; j<pEList->nExpr; j++){ + if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; } } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } } if( 0==(cntTab++) ){ - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; - pSchema = pTab->pSchema; pMatch = pItem; } for(j=0, pCol=pTab->aCol; j<pTab->nCol; j++, pCol++){ @@ -252,17 +300,19 @@ static int lookupName( if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; pMatch = pItem; - pSchema = pTab->pSchema; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; break; } } } - } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->pTab = pMatch->pTab; + pSchema = pExpr->pTab->pSchema; + } + } /* if( pSrcList ) */ #ifndef SQLITE_OMIT_TRIGGER /* If we have not already resolved the name, then maybe @@ -338,7 +388,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; j<pEList->nExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -429,7 +482,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -597,7 +652,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func ){ + }else if( no_such_func && pParse->db->init.busy==0 ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -1033,23 +1088,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } - /* Set up the local name-context to pass to sqlite3ResolveExprNames() to - ** resolve the result-set expression list. - */ - sNC.ncFlags = NC_AllowAgg; - sNC.pSrcList = p->pSrc; - sNC.pNext = pOuterNC; - - /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; i<pEList->nExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } - /* Recursively resolve names in all subqueries */ for(i=0; i<p->pSrc->nSrc; i++){ @@ -1077,6 +1115,23 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; i<pEList->nExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. */ @@ -1104,11 +1159,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries @@ -1229,6 +1283,7 @@ int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1269,6 +1324,7 @@ void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; diff --git a/lib/libsqlite3/src/select.c b/lib/libsqlite3/src/select.c index a90877dc612..f3f14909639 100644 --- a/lib/libsqlite3/src/select.c +++ b/lib/libsqlite3/src/select.c @@ -55,7 +55,7 @@ Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -79,7 +79,7 @@ Select *sqlite3SelectNew( pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->selFlags = selFlags; pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; @@ -1336,8 +1336,6 @@ static int selectColumnsFromExprList( /* Get an appropriate name for the column */ p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS <name>" phrase, use <name> as the name */ zName = sqlite3DbStrDup(db, zName); @@ -1375,6 +1373,9 @@ static int selectColumnsFromExprList( for(j=cnt=0; j<i; j++){ if( sqlite3StrICmp(aCol[j].zName, zName)==0 ){ char *zNewName; + int k; + for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){} + if( zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -1706,6 +1707,8 @@ static int multiSelect( int addr = 0; int nLimit; assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); @@ -2363,7 +2366,8 @@ static int multiSelectOrderBy( }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite3DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); @@ -2545,12 +2549,6 @@ static int multiSelectOrderBy( sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite3VdbeResolveLabel(v, labelEnd); @@ -2962,12 +2960,15 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; + Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; + p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -3160,34 +3161,43 @@ static int flattenSubquery( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: +** Based on the contents of the AggInfo structure indicated by the first +** argument, this function checks if the following are true: ** -** 1. There is a single object in the FROM clause. +** * the query contains just a single aggregate function, +** * the aggregate function is either min() or max(), and +** * the argument to the aggregate function is a column value. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX +** is returned as appropriate. Also, *ppMinMax is set to point to the +** list of arguments passed to the aggregate before returning. +** +** Or, if the conditions above are not met, *ppMinMax is set to 0 and +** WHERE_ORDERBY_NORMAL is returned. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; - - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; +static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + + *ppMinMax = 0; + if( pAggInfo->nFunc==1 ){ + Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ + ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ + + assert( pExpr->op==TK_AGG_FUNCTION ); + if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ + const char *zFunc = pExpr->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + *ppMinMax = pEList; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + *ppMinMax = pEList; + } + } } - return WHERE_ORDERBY_NORMAL; + + assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + return eRet; } /* @@ -3250,6 +3260,69 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3282,14 +3355,16 @@ static int selectExpander(Walker *pWalker, Select *p){ ExprList *pEList; struct SrcList_item *pFrom; sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } - p->selFlags |= SF_Expanded; pTabList = p->pSrc; pEList = p->pEList; @@ -3332,6 +3407,12 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; + if( pTab->nRef==0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ @@ -3367,7 +3448,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** that need expanding. */ for(k=0; k<pEList->nExpr; k++){ - Expr *pE = pEList->a[k].pExpr; + pE = pEList->a[k].pExpr; if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); @@ -3385,10 +3466,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; k<pEList->nExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + pE = a[k].pExpr; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -3403,32 +3492,43 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This expression is a "*" or a "TABLE.*" and needs to be ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ + char *zTName = 0; /* text of name of TABLE */ if( pE->op==TK_DOT ){ assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; - }else{ - zTName = 0; } for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } - tableSeen = 1; for(j=0; j<pTab->nCol; j++){ - Expr *pExpr, *pRight; char *zName = pTab->aCol[j].zName; char *zColname; /* The computed column name */ char *zToFree; /* Malloced string that needs to be freed */ Token sColname; /* Computed column name as a token */ + assert( zName ); + if( zTName && pSub + && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + ){ + continue; + } + /* If a column is marked as 'hidden' (currently only possible ** for virtual tables), do not include it in the expanded ** result-set list. @@ -3437,6 +3537,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert(IsVirtual(pTab)); continue; } + tableSeen = 1; if( i>0 && zTName==0 ){ if( (pFrom->jointype & JT_NATURAL)!=0 @@ -3459,6 +3560,10 @@ static int selectExpander(Walker *pWalker, Select *p){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); zToFree = zColname; @@ -3470,6 +3575,18 @@ static int selectExpander(Walker *pWalker, Select *p){ sColname.z = zColname; sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + if( pSub ){ + pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); + testcase( pX->zSpan==0 ); + }else{ + pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zSpan==0 ); + } + pX->bSpanIsTab = 1; + } sqlite3DbFree(db, zToFree); } } @@ -3522,10 +3639,13 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - w.xSelectCallback = selectExpander; + memset(&w, 0, sizeof(w)); + w.xSelectCallback = convertCompoundSelectToSubquery; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); + w.xSelectCallback = selectExpander; + sqlite3WalkSelect(&w, pSelect); } @@ -3580,9 +3700,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } @@ -3608,6 +3730,7 @@ void sqlite3SelectPrep( sqlite3 *db; if( NEVER(p==0) ) return; db = pParse->db; + if( db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); if( pParse->nErr || db->mallocFailed ) return; @@ -3992,7 +4115,7 @@ int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); @@ -4515,7 +4638,7 @@ int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value @@ -4527,11 +4650,17 @@ int sqlite3Select( ** Refer to code and comments in where.c for details. */ ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); + u8 flag = WHERE_ORDERBY_NORMAL; + + assert( p->pGroupBy==0 ); + assert( flag==0 ); + if( p->pHaving==0 ){ + flag = minMaxQuery(&sAggInfo, &pMinMax); + } + assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - assert( p->pEList->a[0].pExpr->x.pList->nExpr==1 ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; @@ -4687,7 +4816,10 @@ void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ sqlite3ExplainPrintf(pVdbe, "(null-select)"); return; } - while( p->pPrior ) p = p->pPrior; + while( p->pPrior ){ + p->pPrior->pNext = p; + p = p->pPrior; + } sqlite3ExplainPush(pVdbe); while( p ){ explainOneSelect(pVdbe, p); diff --git a/lib/libsqlite3/src/sqlite.h.in b/lib/libsqlite3/src/sqlite.h.in index 48f7381212b..6608823175d 100644 --- a/lib/libsqlite3/src/sqlite.h.in +++ b/lib/libsqlite3/src/sqlite.h.in @@ -283,7 +283,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated ** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close() is called on a [database connection] that still has +** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or ** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], @@ -420,6 +420,8 @@ int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -470,6 +472,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -478,7 +481,19 @@ int sqlite3_exec( #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -718,6 +733,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -854,7 +872,8 @@ struct sqlite3_io_methods { ** it is able to override built-in [PRAGMA] statements. ** ** <li>[[SQLITE_FCNTL_BUSYHANDLER]] -** ^This file-control may be invoked by SQLite on the database file handle +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access ** to the connections busy-handler callback. The argument is of type (void **) ** - an array of two (void *) values. The first (void *) actually points @@ -865,13 +884,24 @@ struct sqlite3_io_methods { ** current operation. ** ** <li>[[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke this file-control to have SQLite generate a +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The ** argument should be a char** which will be filled with the filename ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +** <li>[[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -890,6 +920,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1556,7 +1587,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> -** <dd> ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** <dd> The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -1602,12 +1635,12 @@ struct sqlite3_mem_methods { ** <dt>SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE ** <dd> These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. -** </dl> +** </dd> ** ** [[SQLITE_CONFIG_SQLLOG]] ** <dt>SQLITE_CONFIG_SQLLOG ** <dd>This option is only available if sqlite is compiled with the -** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). ** The second should be of type (void*). The callback is invoked by the library ** in three separate circumstances, identified by the value passed as the @@ -1617,7 +1650,23 @@ struct sqlite3_mem_methods { ** fourth parameter is 1, then the SQL statement that the third parameter ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The -** third parameter is passed NULL In this case. +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.</dd> +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +** <dt>SQLITE_CONFIG_MMAP_SIZE +** <dd>SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1641,6 +1690,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2474,6 +2524,9 @@ int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -2665,7 +2718,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** </ul> ** @@ -3012,7 +3065,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** <li> ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. ** </li> ** ** <li> @@ -3216,6 +3270,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -3983,7 +4040,8 @@ SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4063,14 +4121,17 @@ int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. @@ -4168,7 +4229,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -4967,11 +5028,20 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -4997,11 +5067,11 @@ int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5013,7 +5083,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes @@ -6364,7 +6434,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** ** <table border=1 width=85% align=center> -** <tr><th> createFlag <th> Behaviour when page is not already in cache +** <tr><th> createFlag <th> Behavior when page is not already in cache ** <tr><td> 0 <td> Do not allocate a new page. Return NULL. ** <tr><td> 1 <td> Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. @@ -6794,9 +6864,24 @@ int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); /* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. diff --git a/lib/libsqlite3/src/sqlite3ext.h b/lib/libsqlite3/src/sqlite3ext.h index 5abcde2c8f4..928bb3bad9d 100644 --- a/lib/libsqlite3/src/sqlite3ext.h +++ b/lib/libsqlite3/src/sqlite3ext.h @@ -236,6 +236,20 @@ struct sqlite3_api_routines { int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); int (*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*vsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); }; /* @@ -439,9 +453,32 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen sqlite3_api->blob_reopen #define sqlite3_vtab_config sqlite3_api->vtab_config #define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ diff --git a/lib/libsqlite3/src/sqliteInt.h b/lib/libsqlite3/src/sqliteInt.h index 2b58a808fb4..5950f237d9f 100644 --- a/lib/libsqlite3/src/sqliteInt.h +++ b/lib/libsqlite3/src/sqliteInt.h @@ -66,6 +66,10 @@ # define _GNU_SOURCE #endif +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** Include standard header files as necessary */ @@ -118,11 +122,11 @@ ** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) -#if defined(THREADSAFE) -# define SQLITE_THREADSAFE THREADSAFE -#else -# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ -#endif +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif #endif /* @@ -200,7 +204,8 @@ ** ** See also ticket #2741. */ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) \ + && !defined(__APPLE__) && SQLITE_THREADSAFE # define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ #endif @@ -387,6 +392,7 @@ */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 +# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif /* @@ -534,6 +540,49 @@ extern const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include <TargetConditionals.h> +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -576,6 +625,11 @@ struct BusyHandler { #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) /* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + +/* ** The following value as a destructor means to use sqlite3DbFree(). ** The sqlite3DbFree() routine requires two parameters instead of the ** one parameter that destructors normally want. So we have to introduce @@ -824,6 +878,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -944,7 +999,7 @@ struct sqlite3 { #define SQLITE_SqlTrace 0x00000040 /* Debug print SQL as it executes */ #define SQLITE_VdbeListing 0x00000080 /* Debug listings of VDBE programs */ #define SQLITE_WriteSchema 0x00000100 /* OK to update SQLITE_MASTER */ - /* 0x00000200 Unused */ +#define SQLITE_VdbeAddopTrace 0x00000200 /* Trace sqlite3VdbeAddOp() calls */ #define SQLITE_IgnoreChecks 0x00000400 /* Do not enforce check constraints */ #define SQLITE_ReadUncommitted 0x0000800 /* For shared-cache mode */ #define SQLITE_LegacyFileFmt 0x00001000 /* Create new databases in format 1 */ @@ -973,6 +1028,7 @@ struct sqlite3 { #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ +#define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -1484,20 +1540,20 @@ struct UnpackedRecord { ** element. */ struct Index { - char *zName; /* Name of this index */ - int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ - Table *pTable; /* The SQL table being indexed */ - char *zColAff; /* String defining the affinity of each column */ - Index *pNext; /* The next index associated with the same table */ - Schema *pSchema; /* Schema containing this index */ - u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ - char **azColl; /* Array of collation sequence names for index */ - int nColumn; /* Number of columns in the table used by this index */ - int tnum; /* Page containing root of this index in database file */ - u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ - u8 bUnordered; /* Use this index for == or IN queries only */ + char *zName; /* Name of this index */ + int *aiColumn; /* Which columns are used by this index. 1st is 0 */ + tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + char **azColl; /* Array of collation sequence names for index */ + int tnum; /* DB Page containing root of this index */ + u16 nColumn; /* Number of columns in table used by this index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ #ifdef SQLITE_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ @@ -1771,18 +1827,27 @@ struct Expr { ** list of "ID = expr" items in an UPDATE. A list of expressions can ** also be used as the argument to a function, in which case the a.zName ** field is not used. +** +** By default the Expr.zSpan field holds a human-readable description of +** the expression that is used in the generation of error messages and +** column labels. In this case, Expr.zSpan is typically the text of a +** column expression as it exists in a SELECT statement. However, if +** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name +** of the result column in the form: DATABASE.TABLE.COLUMN. This later +** form is used for name resolution with nested FROM clauses. */ struct ExprList { int nExpr; /* Number of expressions on the list */ int iECursor; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { /* For each expression in the list */ - Expr *pExpr; /* The list of expressions */ - char *zName; /* Token associated with this expression */ - char *zSpan; /* Original text of the expression */ - u8 sortOrder; /* 1 for DESC or 0 for ASC */ - u8 done; /* A flag to indicate when processing is finished */ - u16 iOrderByCol; /* For ORDER BY, column number in result set */ - u16 iAlias; /* Index into Parse.aAlias[] for zName */ + Expr *pExpr; /* The list of expressions */ + char *zName; /* Token associated with this expression */ + char *zSpan; /* Original text of the expression */ + u8 sortOrder; /* 1 for DESC or 0 for ASC */ + unsigned done :1; /* A flag to indicate when processing is finished */ + unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ + u16 iOrderByCol; /* For ORDER BY, column number in result set */ + u16 iAlias; /* Index into Parse.aAlias[] for zName */ } *a; /* Alloc a power of two greater or equal to nExpr */ }; @@ -1950,6 +2015,7 @@ struct WhereLevel { struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ @@ -2049,6 +2115,8 @@ struct NameContext { #define NC_HasAgg 0x02 /* One or more aggregate functions seen */ #define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_AsMaybe 0x10 /* Resolve to AS terms of the result set only + ** if no other resolution is available */ /* ** An instance of the following structure contains all information @@ -2102,6 +2170,7 @@ struct Select { #define SF_UseSorter 0x0040 /* Sort using a sorter */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ #define SF_Materialize 0x0100 /* Force materialization of views */ +#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ /* @@ -2483,6 +2552,8 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ @@ -2517,6 +2588,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ @@ -2814,13 +2886,13 @@ Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, void sqlite3DropIndex(Parse*, SrcList*, int); int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,int,Expr*,Expr*); + Expr*,ExprList*,u16,Expr*,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, int); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) -Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *); +Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); @@ -2888,7 +2960,7 @@ int sqlite3OpenTableAndIndices(Parse*, Table*, int, int); void sqlite3BeginWriteOperation(Parse*, int, int); void sqlite3MultiWrite(Parse*); void sqlite3MayAbort(Parse*); -void sqlite3HaltConstraint(Parse*, int, char*, int); +void sqlite3HaltConstraint(Parse*, int, int, char*, int); Expr *sqlite3ExprDup(sqlite3*,Expr*,int); ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); @@ -3001,8 +3073,11 @@ int sqlite3VarintLen(u64 v); ** x = putVarint32( A, B ); ** */ -#define getVarint32(A,B) (u8)((*(A)<(u8)0x80) ? ((B) = (u32)*(A)),1 : sqlite3GetVarint32((A), (u32 *)&(B))) -#define putVarint32(A,B) (u8)(((u32)(B)<(u32)0x80) ? (*(A) = (unsigned char)(B)),1 : sqlite3PutVarint32((A), (B))) +#define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ + sqlite3PutVarint32((A),(B))) #define getVarint sqlite3GetVarint #define putVarint sqlite3PutVarint @@ -3017,6 +3092,12 @@ void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) +const char *sqlite3ErrName(int); +#endif + const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); @@ -3071,6 +3152,7 @@ void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*); int sqlite3CodeSubselect(Parse *, Expr *, int, int); void sqlite3SelectPrep(Parse*, Select*, NameContext*); +int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); int sqlite3ResolveExprNames(NameContext*, Expr*); void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -3209,8 +3291,10 @@ const char *sqlite3JournalModename(int); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY void sqlite3FkDelete(sqlite3 *, Table*); + int sqlite3FkLocateIndex(Parse*,Table*,FKey*,Index**,int**); #else #define sqlite3FkDelete(a,b) + #define sqlite3FkLocateIndex(a,b,c,d,e) #endif @@ -3235,7 +3319,8 @@ const char *sqlite3JournalModename(int); #define IN_INDEX_ROWID 1 #define IN_INDEX_EPH 2 -#define IN_INDEX_INDEX 3 +#define IN_INDEX_INDEX_ASC 3 +#define IN_INDEX_INDEX_DESC 4 int sqlite3FindInIndex(Parse *, Expr *, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE diff --git a/lib/libsqlite3/src/tclsqlite.c b/lib/libsqlite3/src/tclsqlite.c index 57dab85d4b5..f1bb2921da0 100644 --- a/lib/libsqlite3/src/tclsqlite.c +++ b/lib/libsqlite3/src/tclsqlite.c @@ -1005,7 +1005,7 @@ static int DbTransPostCmd( /* This is a tricky scenario to handle. The most likely cause of an ** error is that the exec() above was an attempt to commit the ** top-level transaction that returned SQLITE_BUSY. Or, less likely, - ** that an IO-error has occured. In either case, throw a Tcl exception + ** that an IO-error has occurred. In either case, throw a Tcl exception ** and try to rollback the transaction. ** ** But it could also be that the user executed one or more BEGIN, @@ -3671,6 +3671,7 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetestschema_Init(Tcl_Interp*); extern int Sqlitetestsse_Init(Tcl_Interp*); extern int Sqlitetesttclvar_Init(Tcl_Interp*); + extern int Sqlitetestfs_Init(Tcl_Interp*); extern int SqlitetestThread_Init(Tcl_Interp*); extern int SqlitetestOnefile_Init(); extern int SqlitetestOsinst_Init(Tcl_Interp*); @@ -3682,8 +3683,6 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitemultiplex_Init(Tcl_Interp*); extern int SqliteSuperlock_Init(Tcl_Interp*); extern int SqlitetestSyscall_Init(Tcl_Interp*); - extern int Sqlitetestfuzzer_Init(Tcl_Interp*); - extern int Sqlitetestwholenumber_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); @@ -3714,6 +3713,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitetest_mutex_Init(interp); Sqlitetestschema_Init(interp); Sqlitetesttclvar_Init(interp); + Sqlitetestfs_Init(interp); SqlitetestThread_Init(interp); SqlitetestOnefile_Init(interp); SqlitetestOsinst_Init(interp); @@ -3725,8 +3725,6 @@ static void init_all(Tcl_Interp *interp){ Sqlitemultiplex_Init(interp); SqliteSuperlock_Init(interp); SqlitetestSyscall_Init(interp); - Sqlitetestfuzzer_Init(interp); - Sqlitetestwholenumber_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/lib/libsqlite3/src/test1.c b/lib/libsqlite3/src/test1.c index bb8d186c121..a638e480add 100644 --- a/lib/libsqlite3/src/test1.c +++ b/lib/libsqlite3/src/test1.c @@ -113,64 +113,8 @@ int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ return TCL_OK; } - -const char *sqlite3TestErrorName(int rc){ - const char *zName = 0; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} -#define t1ErrorName sqlite3TestErrorName +extern const char *sqlite3ErrName(int); +#define t1ErrorName sqlite3ErrName /* ** Convert an sqlite3_stmt* into an sqlite3*. This depends on the @@ -738,6 +682,30 @@ static int sqlite_test_close( } /* +** Usage: sqlite3_close_v2 DB +** +** Closes the database opened by sqlite3_open. +*/ +static int sqlite_test_close_v2( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite3 *db; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " FILENAME\"", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; + rc = sqlite3_close_v2(db); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} + +/* ** Implementation of the x_coalesce() function. ** Return the first argument non-NULL argument. */ @@ -1715,7 +1683,7 @@ static int test_blob_read( if( rc==SQLITE_OK ){ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); }else{ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } Tcl_Free((char *)zBuf); @@ -1765,7 +1733,7 @@ static int test_blob_write( } rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -1791,7 +1759,7 @@ static int test_blob_reopen( rc = sqlite3_blob_reopen(pBlob, iRowid); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -2001,7 +1969,7 @@ static int test_create_function_v2( ); if( rc!=SQLITE_OK ){ Tcl_ResetResult(interp); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -2677,7 +2645,7 @@ static int test_collate( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -3235,7 +3203,7 @@ static int test_bind_text( rc = sqlite3_bind_text(pStmt, idx, value, bytes, SQLITE_TRANSIENT); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -3283,7 +3251,7 @@ static int test_bind_text16( rc = sqlite3_bind_text16(pStmt, idx, (void *)value, bytes, xDel); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -4557,7 +4525,7 @@ static int test_busy_timeout( if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &ms) ) return TCL_ERROR; rc = sqlite3_busy_timeout(db, ms); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_OK; } @@ -5078,7 +5046,7 @@ static int file_control_chunksize_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_CHUNK_SIZE, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5115,7 +5083,7 @@ static int file_control_sizehint_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_SIZE_HINT, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5659,7 +5627,7 @@ static void xLogcallback(void *unused, int err, char *zMsg){ Tcl_Obj *pNew = Tcl_DuplicateObj(logcallback.pObj); Tcl_IncrRefCount(pNew); Tcl_ListObjAppendElement( - 0, pNew, Tcl_NewStringObj(sqlite3TestErrorName(err), -1) + 0, pNew, Tcl_NewStringObj(sqlite3ErrName(err), -1) ); Tcl_ListObjAppendElement(0, pNew, Tcl_NewStringObj(zMsg, -1)); Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT); @@ -5831,6 +5799,31 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX +#include <sys/time.h> +#include <sys/resource.h> + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} +#endif + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -6002,6 +5995,69 @@ static int optimization_control( return TCL_OK; } +typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** load_static_extension DB NAME ... +** +** Load one or more statically linked extensions. +*/ +static int tclLoadStaticExtensionCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); + static const struct { + const char *zExtName; + int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); + } aExtension[] = { + { "amatch", sqlite3_amatch_init }, + { "closure", sqlite3_closure_init }, + { "fuzzer", sqlite3_fuzzer_init }, + { "ieee754", sqlite3_ieee_init }, + { "nextchar", sqlite3_nextchar_init }, + { "regexp", sqlite3_regexp_init }, + { "spellfix", sqlite3_spellfix_init }, + { "wholenumber", sqlite3_wholenumber_init }, + }; + sqlite3 *db; + const char *zName; + int i, j, rc; + char *zErrMsg = 0; + if( objc<3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME ..."); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + for(j=2; j<objc; j++){ + zName = Tcl_GetString(objv[j]); + for(i=0; i<ArraySize(aExtension); i++){ + if( strcmp(zName, aExtension[i].zExtName)==0 ) break; + } + if( i>=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + } + return TCL_OK; +} + + /* ** Register commands with the TCL interpreter. */ @@ -6045,6 +6101,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_get_table_printf", (Tcl_CmdProc*)test_get_table_printf }, #endif { "sqlite3_close", (Tcl_CmdProc*)sqlite_test_close }, + { "sqlite3_close_v2", (Tcl_CmdProc*)sqlite_test_close_v2 }, { "sqlite3_create_function", (Tcl_CmdProc*)test_create_function }, { "sqlite3_create_aggregate", (Tcl_CmdProc*)test_create_aggregate }, { "sqlite_register_test_function", (Tcl_CmdProc*)test_register_func }, @@ -6220,6 +6277,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX + { "getrusage", test_getrusage }, +#endif + { "load_static_extension", tclLoadStaticExtensionCmd }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; @@ -6236,7 +6297,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_DEBUG extern int sqlite3WhereTrace; extern int sqlite3OSTrace; - extern int sqlite3VdbeAddopTrace; extern int sqlite3WalTrace; #endif #ifdef SQLITE_TEST @@ -6299,8 +6359,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); #endif #ifdef SQLITE_DEBUG - Tcl_LinkVar(interp, "sqlite_addop_trace", - (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_where_trace", (char*)&sqlite3WhereTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_os_trace", diff --git a/lib/libsqlite3/src/test2.c b/lib/libsqlite3/src/test2.c index 8acdf6f4efe..d130e9d01b7 100644 --- a/lib/libsqlite3/src/test2.c +++ b/lib/libsqlite3/src/test2.c @@ -19,35 +19,7 @@ #include <string.h> #include <ctype.h> -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3ErrName(int); /* ** Page size and reserved size used for testing. @@ -87,7 +59,7 @@ static int pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); @@ -119,7 +91,7 @@ static int pager_close( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -146,7 +118,7 @@ static int pager_rollback( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -173,12 +145,12 @@ static int pager_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -205,7 +177,7 @@ static int pager_stmt_begin( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -233,7 +205,7 @@ static int pager_stmt_rollback( rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -260,7 +232,7 @@ static int pager_stmt_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -353,7 +325,7 @@ static int page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); @@ -507,7 +479,7 @@ static int page_write( pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -556,7 +528,7 @@ static int fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0); sqlite3_free(zFile); return TCL_ERROR; } @@ -566,7 +538,7 @@ static int fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; diff --git a/lib/libsqlite3/src/test3.c b/lib/libsqlite3/src/test3.c index e460c42e46c..e3ed310c810 100644 --- a/lib/libsqlite3/src/test3.c +++ b/lib/libsqlite3/src/test3.c @@ -19,31 +19,7 @@ #include <stdlib.h> #include <string.h> -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3ErrName(int); /* ** A bogus sqlite3 connection structure for use in the btree @@ -89,7 +65,7 @@ static int btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); @@ -119,7 +95,7 @@ static int btree_close( pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } nRefSqlite3--; @@ -156,7 +132,7 @@ static int btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -250,7 +226,7 @@ static int btree_cursor( sqlite3BtreeLeave(pBt); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); @@ -285,7 +261,7 @@ static int btree_close_cursor( sqlite3BtreeLeave(pBt); ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return SQLITE_OK; @@ -319,7 +295,7 @@ static int btree_next( rc = sqlite3BtreeNext(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); @@ -354,7 +330,7 @@ static int btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); diff --git a/lib/libsqlite3/src/test4.c b/lib/libsqlite3/src/test4.c index 5b4c5a1c178..a6375c7cc40 100644 --- a/lib/libsqlite3/src/test4.c +++ b/lib/libsqlite3/src/test4.c @@ -20,6 +20,8 @@ #include <sched.h> #include <ctype.h> +extern const char *sqlite3ErrName(int); + /* ** Each thread is controlled by an instance of the following ** structure. @@ -372,34 +374,7 @@ static int tcl_thread_result( return TCL_ERROR; } thread_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/lib/libsqlite3/src/test6.c b/lib/libsqlite3/src/test6.c index f511be9def8..c151ea42989 100644 --- a/lib/libsqlite3/src/test6.c +++ b/lib/libsqlite3/src/test6.c @@ -87,7 +87,7 @@ typedef struct WriteBuffer WriteBuffer; ** an aligned write() of an integer number of 512 byte regions, then ** option (3) above is never selected. Instead, each 512 byte region ** is either correctly written or left completely untouched. Similar -** logic governs the behaviour if any of the other ATOMICXXX flags +** logic governs the behavior if any of the other ATOMICXXX flags ** is set. ** ** If either the IOCAP_SAFEAPPEND or IOCAP_SEQUENTIAL flags are set diff --git a/lib/libsqlite3/src/test7.c b/lib/libsqlite3/src/test7.c index 852cd1db5e8..3cd4a224d7e 100644 --- a/lib/libsqlite3/src/test7.c +++ b/lib/libsqlite3/src/test7.c @@ -376,6 +376,8 @@ static int tcl_client_colname( return TCL_OK; } +extern const char *sqlite3ErrName(int); + /* ** Usage: client_result ID ** @@ -403,34 +405,7 @@ static int tcl_client_result( return TCL_ERROR; } client_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/lib/libsqlite3/src/test8.c b/lib/libsqlite3/src/test8.c index 53cb149a04b..c5739332d46 100644 --- a/lib/libsqlite3/src/test8.c +++ b/lib/libsqlite3/src/test8.c @@ -1300,7 +1300,7 @@ static sqlite3_module echoModuleV2 = { ** Decode a pointer to an sqlite3 object. */ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); static void moduleDestroy(void *p){ sqlite3_free(p); @@ -1340,7 +1340,7 @@ static int register_echo_module( ); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1370,29 +1370,6 @@ static int declare_vtab( return TCL_OK; } -#include "test_spellfix.c" - -/* -** Register the spellfix virtual table module. -*/ -static int register_spellfix_module( - ClientData clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - - sqlite3Spellfix1Register(db); - return TCL_OK; -} - #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /* @@ -1406,7 +1383,6 @@ int Sqlitetest8_Init(Tcl_Interp *interp){ void *clientData; } aObjCmd[] = { { "register_echo_module", register_echo_module, 0 }, - { "register_spellfix_module", register_spellfix_module, 0 }, { "sqlite3_declare_vtab", declare_vtab, 0 }, }; int i; diff --git a/lib/libsqlite3/src/test_async.c b/lib/libsqlite3/src/test_async.c index c760eea1fa2..b0b943185bf 100644 --- a/lib/libsqlite3/src/test_async.c +++ b/lib/libsqlite3/src/test_async.c @@ -23,8 +23,8 @@ #include "sqlite3.h" #include <assert.h> -/* From test1.c */ -const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); struct TestAsyncGlobal { @@ -60,7 +60,7 @@ static int testAsyncInit( rc = sqlite3async_initialize(zParent, isDefault); if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } return TCL_OK; @@ -208,7 +208,7 @@ static int testAsyncControl( } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/lib/libsqlite3/src/test_autoext.c b/lib/libsqlite3/src/test_autoext.c index 6b1e297ab7c..b5013f3173a 100644 --- a/lib/libsqlite3/src/test_autoext.c +++ b/lib/libsqlite3/src/test_autoext.c @@ -15,7 +15,7 @@ #include "sqlite3ext.h" #ifndef SQLITE_OMIT_LOAD_EXTENSION -static SQLITE_EXTENSION_INIT1 +SQLITE_EXTENSION_INIT1 /* ** The sqr() SQL function returns the square of its input value. diff --git a/lib/libsqlite3/src/test_backup.c b/lib/libsqlite3/src/test_backup.c index 272713785a1..e967424a290 100644 --- a/lib/libsqlite3/src/test_backup.c +++ b/lib/libsqlite3/src/test_backup.c @@ -17,9 +17,11 @@ #include <sqlite3.h> #include <assert.h> +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + /* These functions are implemented in test1.c. */ -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -const char *sqlite3TestErrorName(int); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); static int backupTestCmd( ClientData clientData, @@ -70,7 +72,7 @@ static int backupTestCmd( Tcl_DeleteCommand(interp, zCmdName); rc = sqlite3_backup_finish(p); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } @@ -80,7 +82,7 @@ static int backupTestCmd( return TCL_ERROR; } rc = sqlite3_backup_step(p, nPage); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } diff --git a/lib/libsqlite3/src/test_config.c b/lib/libsqlite3/src/test_config.c index f79b455a57f..534727a0804 100644 --- a/lib/libsqlite3/src/test_config.c +++ b/lib/libsqlite3/src/test_config.c @@ -57,7 +57,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","0",TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_CURDIR +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); @@ -87,6 +87,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif +#if SQLITE_MAX_MMAP_SIZE>0 + Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else @@ -395,11 +401,7 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "memorymanage", "0", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_OMIT_MERGE_SORT - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "0", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); -#endif +Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); #ifdef SQLITE_OMIT_OR_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY); diff --git a/lib/libsqlite3/src/test_fs.c b/lib/libsqlite3/src/test_fs.c new file mode 100644 index 00000000000..478cad80b16 --- /dev/null +++ b/lib/libsqlite3/src/test_fs.c @@ -0,0 +1,333 @@ +/* +** 2013 Jan 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing the virtual table interfaces. This code +** is not included in the SQLite library. It is used for automated +** testing of the SQLite library. +** +** The FS virtual table is created as follows: +** +** CREATE VIRTUAL TABLE tbl USING fs(idx); +** +** where idx is the name of a table in the db with 2 columns. The virtual +** table also has two columns - file path and file contents. +** +** The first column of table idx must be an IPK, and the second contains file +** paths. For example: +** +** CREATE TABLE idx(id INTEGER PRIMARY KEY, path TEXT); +** INSERT INTO idx VALUES(4, '/etc/passwd'); +** +** Adding the row to the idx table automatically creates a row in the +** virtual table with rowid=4, path=/etc/passwd and a text field that +** contains data read from file /etc/passwd on disk. +*/ +#include "sqliteInt.h" +#include "tcl.h" + +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#if SQLITE_OS_UNIX +# include <unistd.h> +#endif +#if SQLITE_OS_WIN +# include <io.h> +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +typedef struct fs_vtab fs_vtab; +typedef struct fs_cursor fs_cursor; + +/* +** A fs virtual-table object +*/ +struct fs_vtab { + sqlite3_vtab base; + sqlite3 *db; + char *zDb; /* Name of db containing zTbl */ + char *zTbl; /* Name of docid->file map table */ +}; + +/* A fs cursor object */ +struct fs_cursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; + char *zBuf; + int nBuf; + int nAlloc; +}; + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the fs virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fs") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> other module argument fields. +*/ +static int fsConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fs_vtab *pVtab; + int nByte; + const char *zTbl; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("wrong number of arguments"); + return SQLITE_ERROR; + } + zTbl = argv[3]; + + nByte = sizeof(fs_vtab) + (int)strlen(zTbl) + 1 + (int)strlen(zDb) + 1; + pVtab = (fs_vtab *)sqlite3MallocZero( nByte ); + if( !pVtab ) return SQLITE_NOMEM; + + pVtab->zTbl = (char *)&pVtab[1]; + pVtab->zDb = &pVtab->zTbl[strlen(zTbl)+1]; + pVtab->db = db; + memcpy(pVtab->zTbl, zTbl, strlen(zTbl)); + memcpy(pVtab->zDb, zDb, strlen(zDb)); + *ppVtab = &pVtab->base; + sqlite3_declare_vtab(db, "CREATE TABLE xyz(path TEXT, data TEXT)"); + + return SQLITE_OK; +} +/* Note that for this virtual table, the xCreate and xConnect +** methods are identical. */ + +static int fsDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} +/* The xDisconnect and xDestroy methods are also the same */ + +/* +** Open a new fs cursor. +*/ +static int fsOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fs_cursor *pCur; + pCur = sqlite3MallocZero(sizeof(fs_cursor)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Close a fs cursor. +*/ +static int fsClose(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor *)cur; + sqlite3_finalize(pCur->pStmt); + sqlite3_free(pCur->zBuf); + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int fsNext(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor *)cur; + int rc; + + rc = sqlite3_step(pCur->pStmt); + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; + + return rc; +} + +static int fsFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + int rc; + fs_cursor *pCur = (fs_cursor *)pVtabCursor; + fs_vtab *p = (fs_vtab *)(pVtabCursor->pVtab); + + assert( (idxNum==0 && argc==0) || (idxNum==1 && argc==1) ); + if( idxNum==1 ){ + char *zStmt = sqlite3_mprintf( + "SELECT * FROM %Q.%Q WHERE rowid=?", p->zDb, p->zTbl); + if( !zStmt ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pCur->pStmt, 0); + sqlite3_free(zStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_value(pCur->pStmt, 1, argv[0]); + } + }else{ + char *zStmt = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zTbl); + if( !zStmt ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pCur->pStmt, 0); + sqlite3_free(zStmt); + } + + if( rc==SQLITE_OK ){ + rc = fsNext(pVtabCursor); + } + return rc; +} + +static int fsColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + fs_cursor *pCur = (fs_cursor*)cur; + + assert( i==0 || i==1 ); + if( i==0 ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pStmt, 0)); + }else{ + const char *zFile = (const char *)sqlite3_column_text(pCur->pStmt, 1); + struct stat sbuf; + int fd; + + fd = open(zFile, O_RDONLY); + if( fd<0 ) return SQLITE_IOERR; + fstat(fd, &sbuf); + + if( sbuf.st_size>=pCur->nAlloc ){ + int nNew = sbuf.st_size*2; + char *zNew; + if( nNew<1024 ) nNew = 1024; + + zNew = sqlite3Realloc(pCur->zBuf, nNew); + if( zNew==0 ){ + close(fd); + return SQLITE_NOMEM; + } + pCur->zBuf = zNew; + pCur->nAlloc = nNew; + } + + read(fd, pCur->zBuf, sbuf.st_size); + close(fd); + pCur->nBuf = sbuf.st_size; + pCur->zBuf[pCur->nBuf] = '\0'; + + sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +static int fsRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fs_cursor *pCur = (fs_cursor*)cur; + *pRowid = sqlite3_column_int64(pCur->pStmt, 0); + return SQLITE_OK; +} + +static int fsEof(sqlite3_vtab_cursor *cur){ + fs_cursor *pCur = (fs_cursor*)cur; + return (sqlite3_data_count(pCur->pStmt)==0); +} + +static int fsBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + + for(ii=0; ii<pIdxInfo->nConstraint; ii++){ + struct sqlite3_index_constraint const *pCons = &pIdxInfo->aConstraint[ii]; + if( pCons->iColumn<0 && pCons->usable + && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + struct sqlite3_index_constraint_usage *pUsage; + pUsage = &pIdxInfo->aConstraintUsage[ii]; + pUsage->omit = 0; + pUsage->argvIndex = 1; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 1.0; + break; + } + } + + return SQLITE_OK; +} + +/* +** A virtual table module that provides read-only access to a +** Tcl global variable namespace. +*/ +static sqlite3_module fsModule = { + 0, /* iVersion */ + fsConnect, + fsConnect, + fsBestIndex, + fsDisconnect, + fsDisconnect, + fsOpen, /* xOpen - open a cursor */ + fsClose, /* xClose - close a cursor */ + fsFilter, /* xFilter - configure scan constraints */ + fsNext, /* xNext - advance a cursor */ + fsEof, /* xEof - check for end of scan */ + fsColumn, /* xColumn - read data */ + fsRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/* +** Decode a pointer to an sqlite3 object. +*/ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); + +/* +** Register the echo virtual table module. +*/ +static int register_fs_module( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_create_module(db, "fs", &fsModule, (void *)interp); +#endif + return TCL_OK; +} + +#endif + + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetestfs_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + void *clientData; + } aObjCmd[] = { + { "register_fs_module", register_fs_module, 0 }, + }; + int i; + for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ + Tcl_CreateObjCommand(interp, aObjCmd[i].zName, + aObjCmd[i].xProc, aObjCmd[i].clientData, 0); + } +#endif + return TCL_OK; +} diff --git a/lib/libsqlite3/src/test_intarray.c b/lib/libsqlite3/src/test_intarray.c index 6ea77561768..f5c3d9e4059 100644 --- a/lib/libsqlite3/src/test_intarray.c +++ b/lib/libsqlite3/src/test_intarray.c @@ -278,7 +278,7 @@ int sqlite3_intarray_bind( extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); extern void *sqlite3TestTextToPtr(const char*); extern int sqlite3TestMakePointerStr(Tcl_Interp*, char *zPtr, void*); -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** sqlite3_intarray_create DB NAME @@ -309,7 +309,7 @@ static int test_intarray_create( #endif if( rc!=SQLITE_OK ){ assert( pArray==0 ); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } sqlite3TestMakePointerStr(interp, zPtr, pArray); @@ -352,7 +352,7 @@ static int test_intarray_bind( } rc = sqlite3_intarray_bind(pArray, n, a, sqlite3_free); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } #endif diff --git a/lib/libsqlite3/src/test_malloc.c b/lib/libsqlite3/src/test_malloc.c index 33de859d274..cf98a8fb218 100644 --- a/lib/libsqlite3/src/test_malloc.c +++ b/lib/libsqlite3/src/test_malloc.c @@ -234,14 +234,14 @@ static int faultsimInstall(int install){ #ifdef SQLITE_TEST /* -** This function is implemented in test1.c. Returns a pointer to a static +** This function is implemented in main.c. Returns a pointer to a static ** buffer containing the symbolic SQLite error code that corresponds to ** the least-significant 8-bits of the integer passed as an argument. ** For example: ** -** sqlite3TestErrorName(1) -> "SQLITE_ERROR" +** sqlite3ErrName(1) -> "SQLITE_ERROR" */ -const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** Transform pointers to text and back again @@ -1072,7 +1072,7 @@ static int test_db_config_lookaside( sqlite3 *db; int bufid; static char azBuf[2][10000]; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "BUFID SIZE COUNT"); return TCL_ERROR; @@ -1126,7 +1126,7 @@ static int test_config_heap( rc = sqlite3_config(SQLITE_CONFIG_HEAP, zBuf, nByte, nMinAlloc); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1143,7 +1143,7 @@ static int test_config_error( Tcl_Obj *CONST objv[] ){ sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=2 && objc!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "[DB]"); @@ -1192,7 +1192,7 @@ static int test_config_uri( } rc = sqlite3_config(SQLITE_CONFIG_URI, bOpenUri); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1221,7 +1221,7 @@ static int test_config_cis( } rc = sqlite3_config(SQLITE_CONFIG_COVERING_INDEX_SCAN, bUseCis); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1335,7 +1335,7 @@ static int test_db_status( int i, op, resetFlag; const char *zOpName; sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); static const struct { const char *zName; int op; @@ -1401,7 +1401,7 @@ static int test_install_malloc_faultsim( return TCL_ERROR; } rc = faultsimInstall(isInstall); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1419,7 +1419,7 @@ static int test_install_memsys3( const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); rc = sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetMemsys3()); #endif - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/lib/libsqlite3/src/test_multiplex.c b/lib/libsqlite3/src/test_multiplex.c index 23df347ded5..624541b32a1 100644 --- a/lib/libsqlite3/src/test_multiplex.c +++ b/lib/libsqlite3/src/test_multiplex.c @@ -60,7 +60,7 @@ /* ** These should be defined to be the same as the values in -** sqliteInt.h. They are defined seperately here so that +** sqliteInt.h. They are defined separately here so that ** the multiplex VFS shim can be built as a loadable ** module. */ @@ -1183,7 +1183,7 @@ int sqlite3_multiplex_shutdown(void){ /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST #include <tcl.h> -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1212,7 +1212,7 @@ static int test_multiplex_initialize( /* Call sqlite3_multiplex_initialize() */ rc = sqlite3_multiplex_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1237,7 +1237,7 @@ static int test_multiplex_shutdown( /* Call sqlite3_multiplex_shutdown() */ rc = sqlite3_multiplex_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1355,7 +1355,7 @@ static int test_multiplex_control( } rc = sqlite3_file_control(db, Tcl_GetString(objv[2]), aSub[idx].op, pArg); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return (rc==SQLITE_OK) ? TCL_OK : TCL_ERROR; } diff --git a/lib/libsqlite3/src/test_mutex.c b/lib/libsqlite3/src/test_mutex.c index 0bb74375d8f..c9b4a29ab75 100644 --- a/lib/libsqlite3/src/test_mutex.c +++ b/lib/libsqlite3/src/test_mutex.c @@ -19,8 +19,8 @@ #include <assert.h> #include <string.h> -/* defined in test1.c */ -const char *sqlite3TestErrorName(int); +/* defined in main.c */ +extern const char *sqlite3ErrName(int); /* A countable mutex */ struct sqlite3_mutex { @@ -148,7 +148,7 @@ static int test_shutdown( } rc = sqlite3_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -169,7 +169,7 @@ static int test_initialize( } rc = sqlite3_initialize(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -230,7 +230,7 @@ static int test_install_mutex_counters( g.isInstalled = isInstall; } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -354,7 +354,7 @@ static int test_config( } rc = sqlite3_config(i); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/lib/libsqlite3/src/test_quota.c b/lib/libsqlite3/src/test_quota.c index 166a512f18c..e590996ca4b 100644 --- a/lib/libsqlite3/src/test_quota.c +++ b/lib/libsqlite3/src/test_quota.c @@ -1295,7 +1295,7 @@ int sqlite3_quota_remove(const char *zFilename){ if( pGroup ){ for(pFile=pGroup->pFiles; pFile && rc==SQLITE_OK; pFile=pNextFile){ pNextFile = pFile->pNext; - diff = memcmp(zFull, pFile->zFilename, nFull); + diff = strncmp(zFull, pFile->zFilename, nFull); if( diff==0 && ((c = pFile->zFilename[nFull])==0 || c=='/' || c=='\\') ){ if( pFile->nRef ){ pFile->deleteOnClose = 1; @@ -1325,7 +1325,7 @@ struct TclQuotaCallback { Tcl_Obj *pScript; /* Script to be run */ }; -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1407,7 +1407,7 @@ static int test_quota_initialize( /* Call sqlite3_quota_initialize() */ rc = sqlite3_quota_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1430,7 +1430,7 @@ static int test_quota_shutdown( /* Call sqlite3_quota_shutdown() */ rc = sqlite3_quota_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1485,7 +1485,7 @@ static int test_quota_set( /* Invoke sqlite3_quota_set() */ rc = sqlite3_quota_set(zPattern, iLimit, xCallback, (void*)p, xDestroy); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1511,7 +1511,7 @@ static int test_quota_file( /* Invoke sqlite3_quota_file() */ rc = sqlite3_quota_file(zFilename); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } diff --git a/lib/libsqlite3/src/test_rtree.c b/lib/libsqlite3/src/test_rtree.c index d3c9e0cb3de..f54ae9b063b 100644 --- a/lib/libsqlite3/src/test_rtree.c +++ b/lib/libsqlite3/src/test_rtree.c @@ -254,7 +254,7 @@ static int register_cube_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -264,7 +264,7 @@ static int register_cube_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "cube", cube_geom, (void *)&gHere); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } @@ -282,7 +282,7 @@ static int register_circle_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -292,7 +292,7 @@ static int register_circle_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "circle", circle_geom, 0); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } diff --git a/lib/libsqlite3/src/test_sqllog.c b/lib/libsqlite3/src/test_sqllog.c index 7cb570bcf22..4aa68b7c429 100644 --- a/lib/libsqlite3/src/test_sqllog.c +++ b/lib/libsqlite3/src/test_sqllog.c @@ -13,26 +13,36 @@ ** OVERVIEW ** ** This file contains experimental code used to record data from live -** SQLite applications that may be useful for offline analysis. Specifically: +** SQLite applications that may be useful for offline analysis. +** Specifically, this module can be used to capture the following +** information: ** ** 1) The initial contents of all database files opened by the ** application, and ** ** 2) All SQL statements executed by the application. ** +** The captured information can then be used to run (for example) +** performance analysis looking for slow queries or to look for +** optimization opportunities in either the application or in SQLite +** itself. +** ** USAGE ** ** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG -** pre-processor symbol defined and this file linked into the application -** somehow. +** pre-processor symbol defined and this file linked into the application. +** One way to link this file into the application is to append the content +** of this file onto the end of the "sqlite3.c" amalgamation and then +** recompile the application as normal except with the addition of the +** -DSQLITE_ENABLE_SQLLOG option. ** ** At runtime, logging is enabled by setting environment variable ** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged -** data. The directory must already exist. +** data. The logging directory must already exist. ** ** Usually, if the application opens the same database file more than once ** (either by attaching it or by using more than one database handle), only -** a single copy is made. This behaviour may be overridden (so that a +** a single copy is made. This behavior may be overridden (so that a ** separate copy is taken each time the database file is opened or attached) ** by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0. ** @@ -57,14 +67,16 @@ ** logging proceeds as expected. Errors are logged by calling sqlite3_log(). */ +#ifndef _SQLITE3_H_ #include "sqlite3.h" -#include "stdio.h" -#include "stdlib.h" -#include "string.h" -#include "assert.h" +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> -#include "sys/types.h" -#include "unistd.h" +#include <sys/types.h> +#include <unistd.h> static int getProcessId(void){ #if SQLITE_OS_WIN return (int)_getpid(); @@ -73,7 +85,7 @@ static int getProcessId(void){ #endif } - +/* Names of environment variables to be used */ #define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" #define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" @@ -86,6 +98,9 @@ static int getProcessId(void){ */ #define MAX_CONNECTIONS 256 +/* There is one instance of this object for each SQLite database connection +** that is being logged. +*/ struct SLConn { int isErr; /* True if an error has occurred */ sqlite3 *db; /* Connection handle */ @@ -93,7 +108,9 @@ struct SLConn { FILE *fd; /* File descriptor for log file */ }; -struct SLGlobal { +/* This object is a singleton that keeps track of all data loggers. +*/ +static struct SLGlobal { /* Protected by MUTEX_STATIC_MASTER */ sqlite3_mutex *mutex; /* Recursive mutex */ int nConn; /* Size of aConn[] array */ @@ -388,6 +405,24 @@ static void testSqllogStmt(struct SLConn *p, const char *zSql){ /* ** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +** +** The eType parameter has the following values: +** +** 0: Opening a new database connection. zSql is the name of the +** file being opened. db is a pointer to the newly created database +** connection. +** +** 1: An SQL statement has run to completion. zSql is the text of the +** SQL statement with all parameters expanded to their actual values. +** +** 2: Closing a database connection. zSql is NULL. The db pointer to +** the database connection being closed has already been shut down +** and cannot be used for any further SQL. +** +** The pCtx parameter is a copy of the pointer that was originally passed +** into the sqlite3_config(SQLITE_CONFIG_SQLLOG) statement. In this +** particular implementation, pCtx is always a pointer to the +** sqllogglobal global variable define above. */ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ struct SLConn *p = 0; diff --git a/lib/libsqlite3/src/test_syscall.c b/lib/libsqlite3/src/test_syscall.c index d484f22db4b..7c0873c16d8 100644 --- a/lib/libsqlite3/src/test_syscall.c +++ b/lib/libsqlite3/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -69,18 +69,19 @@ ** the xNextSystemCall() VFS method. */ +#include "sqliteInt.h" #include "sqlite3.h" #include "tcl.h" #include <stdlib.h> #include <string.h> #include <assert.h> -#include "sqliteInt.h" #if SQLITE_OS_UNIX -/* From test1.c */ -extern const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); +#include <sys/mman.h> #include <sys/types.h> #include <errno.h> @@ -106,7 +107,8 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); - +static void *ts_mmap(void *, size_t, int, int, int, off_t); +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -131,6 +133,8 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +156,8 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +383,31 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, @@ -467,7 +498,7 @@ static int test_syscall_reset( } } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/lib/libsqlite3/src/test_thread.c b/lib/libsqlite3/src/test_thread.c index ae62de8262d..2f9363b7501 100644 --- a/lib/libsqlite3/src/test_thread.c +++ b/lib/libsqlite3/src/test_thread.c @@ -60,12 +60,14 @@ static Tcl_ObjCmdProc blocking_prepare_v2_proc; int Sqlitetest1_Init(Tcl_Interp *); int Sqlite3_Init(Tcl_Interp *); +/* Functions from main.c */ +extern const char *sqlite3ErrName(int); + /* Functions from test1.c */ -void *sqlite3TestTextToPtr(const char *); -const char *sqlite3TestErrorName(int); -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); -int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); +extern void *sqlite3TestTextToPtr(const char *); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); +extern int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); +extern int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); /* ** Handler for events of type EvalEvent. @@ -559,7 +561,7 @@ static int blocking_step_proc( pStmt = (sqlite3_stmt*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); rc = sqlite3_blocking_step(pStmt); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), 0); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), 0); return TCL_OK; } @@ -606,7 +608,7 @@ static int blocking_prepare_v2_proc( } if( rc!=SQLITE_OK ){ assert( pStmt==0 ); - sprintf(zBuf, "%s ", (char *)sqlite3TestErrorName(rc)); + sprintf(zBuf, "%s ", (char *)sqlite3ErrName(rc)); Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); return TCL_ERROR; } diff --git a/lib/libsqlite3/src/test_vfs.c b/lib/libsqlite3/src/test_vfs.c index 93c556b56e0..fcd577439cf 100644 --- a/lib/libsqlite3/src/test_vfs.c +++ b/lib/libsqlite3/src/test_vfs.c @@ -125,8 +125,9 @@ struct Testvfs { #define TESTVFS_ACCESS_MASK 0x00004000 #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 +#define TESTVFS_UNLOCK_MASK 0x00020000 -#define TESTVFS_ALL_MASK 0x0001FFFF +#define TESTVFS_ALL_MASK 0x0003FFFF #define TESTVFS_MAX_PAGES 1024 @@ -265,7 +266,8 @@ static void tvfsExecTcl( const char *zMethod, Tcl_Obj *arg1, Tcl_Obj *arg2, - Tcl_Obj *arg3 + Tcl_Obj *arg3, + Tcl_Obj *arg4 ){ int rc; /* Return code from Tcl_EvalObj() */ Tcl_Obj *pEval; @@ -282,6 +284,7 @@ static void tvfsExecTcl( if( arg1 ) Tcl_ListObjAppendElement(p->interp, pEval, arg1); if( arg2 ) Tcl_ListObjAppendElement(p->interp, pEval, arg2); if( arg3 ) Tcl_ListObjAppendElement(p->interp, pEval, arg3); + if( arg4 ) Tcl_ListObjAppendElement(p->interp, pEval, arg4); rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); if( rc!=TCL_OK ){ @@ -302,7 +305,7 @@ static int tvfsClose(sqlite3_file *pFile){ if( p->pScript && p->mask&TESTVFS_CLOSE_MASK ){ tvfsExecTcl(p, "xClose", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); } @@ -333,7 +336,7 @@ static int tvfsRead( Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_READ_MASK ){ tvfsExecTcl(p, "xRead", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -362,7 +365,7 @@ static int tvfsWrite( if( p->pScript && p->mask&TESTVFS_WRITE_MASK ){ tvfsExecTcl(p, "xWrite", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewWideIntObj(iOfst) + Tcl_NewWideIntObj(iOfst), Tcl_NewIntObj(iAmt) ); tvfsResultCode(p, &rc); } @@ -390,7 +393,7 @@ static int tvfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->pScript && p->mask&TESTVFS_TRUNCATE_MASK ){ tvfsExecTcl(p, "xTruncate", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -431,7 +434,7 @@ static int tvfsSync(sqlite3_file *pFile, int flags){ tvfsExecTcl(p, "xSync", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewStringObj(zFlags, -1) + Tcl_NewStringObj(zFlags, -1), 0 ); tvfsResultCode(p, &rc); } @@ -465,8 +468,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ ** Unlock an tvfs-file. */ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsUnlock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ + return SQLITE_IOERR_UNLOCK; + } + return sqlite3OsUnlock(pFd->pReal, eLock); } /* @@ -578,7 +585,7 @@ static int tvfsOpen( z += strlen(z) + 1; } } - tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0); + tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0, 0); Tcl_DecrRefCount(pArg); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; @@ -635,7 +642,7 @@ static int tvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ if( p->pScript && p->mask&TESTVFS_DELETE_MASK ){ tvfsExecTcl(p, "xDelete", - Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0, 0 ); tvfsResultCode(p, &rc); } @@ -663,7 +670,7 @@ static int tvfsAccess( if( flags==SQLITE_ACCESS_READWRITE ) zArg = "SQLITE_ACCESS_READWRITE"; if( flags==SQLITE_ACCESS_READ ) zArg = "SQLITE_ACCESS_READ"; tvfsExecTcl(p, "xAccess", - Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0, 0 ); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; @@ -691,7 +698,7 @@ static int tvfsFullPathname( Testvfs *p = (Testvfs *)pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_FULLPATHNAME_MASK ){ int rc; - tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0); + tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } @@ -771,7 +778,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){ */ Tcl_ResetResult(p->interp); if( p->pScript && p->mask&TESTVFS_SHMOPEN_MASK ){ - tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0); + tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } @@ -841,7 +848,7 @@ static int tvfsShmMap( Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite)); tvfsExecTcl(p, "xShmMap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg, 0 ); tvfsResultCode(p, &rc); Tcl_DecrRefCount(pArg); @@ -891,7 +898,7 @@ static int tvfsShmLock( } tvfsExecTcl(p, "xShmLock", Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, - Tcl_NewStringObj(zLock, -1) + Tcl_NewStringObj(zLock, -1), 0 ); tvfsResultCode(p, &rc); } @@ -937,7 +944,7 @@ static void tvfsShmBarrier(sqlite3_file *pFile){ if( p->pScript && p->mask&TESTVFS_SHMBARRIER_MASK ){ tvfsExecTcl(p, "xShmBarrier", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); } } @@ -961,7 +968,7 @@ static int tvfsShmUnmap( if( p->pScript && p->mask&TESTVFS_SHMCLOSE_MASK ){ tvfsExecTcl(p, "xShmUnmap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } @@ -1099,6 +1106,7 @@ static int testvfs_obj_cmd( { "xClose", TESTVFS_CLOSE_MASK }, { "xAccess", TESTVFS_ACCESS_MASK }, { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; diff --git a/lib/libsqlite3/src/update.c b/lib/libsqlite3/src/update.c index 96ba4df83dd..3ab1ab2a4b6 100644 --- a/lib/libsqlite3/src/update.c +++ b/lib/libsqlite3/src/update.c @@ -208,6 +208,7 @@ void sqlite3Update( } if( j>=pTab->nCol ){ if( sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ @@ -220,7 +221,8 @@ void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -458,7 +460,7 @@ void sqlite3Update( /* The row-trigger may have deleted the row being updated. In this ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated + ** required. This behavior - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the ** documentation. */ diff --git a/lib/libsqlite3/src/util.c b/lib/libsqlite3/src/util.c index 5cf8ebacb5d..d83a63015fd 100644 --- a/lib/libsqlite3/src/util.c +++ b/lib/libsqlite3/src/util.c @@ -261,7 +261,7 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ */ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT - int incr = (enc==SQLITE_UTF8?1:2); + int incr; const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ @@ -272,10 +272,22 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + int nonNum = 0; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ - if( enc==SQLITE_UTF16BE ) z++; + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + int i; + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i<length && z[i]==0; i+=2){} + nonNum = i<length; + zEnd = z+i+enc-3; + z += (enc&1); + } /* skip leading spaces */ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; @@ -408,7 +420,7 @@ do_atof_calc: *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z>=zEnd && nDigits>0 && eValid; + return z>=zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -457,21 +469,33 @@ static int compare2pow63(const char *zNum, int incr){ ** signed 64-bit integer, its negative -9223372036854665808 can be. ** ** If zNum is too big for a 64-bit integer and is not -** 9223372036854665808 then return 1. +** 9223372036854665808 or if zNum contains any non-numeric text, +** then return 1. ** ** length is the number of bytes in the string (bytes, not characters). ** The string is not necessarily zero-terminated. The encoding is ** given by enc. */ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ - int incr = (enc==SQLITE_UTF8?1:2); + int incr; u64 u = 0; int neg = 0; /* assume positive */ int i; int c = 0; + int nonNum = 0; const char *zStart; const char *zEnd = zNum + length; - if( enc==SQLITE_UTF16BE ) zNum++; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i<length && zNum[i]==0; i+=2){} + nonNum = i<length; + zEnd = zNum+i+enc-3; + zNum += (enc&1); + } while( zNum<zEnd && sqlite3Isspace(*zNum) ) zNum+=incr; if( zNum<zEnd ){ if( *zNum=='-' ){ @@ -496,7 +520,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr ){ + if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr || nonNum ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; diff --git a/lib/libsqlite3/src/vacuum.c b/lib/libsqlite3/src/vacuum.c index 7ed2478326c..4afb2cca64d 100644 --- a/lib/libsqlite3/src/vacuum.c +++ b/lib/libsqlite3/src/vacuum.c @@ -289,6 +289,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); diff --git a/lib/libsqlite3/src/vdbe.c b/lib/libsqlite3/src/vdbe.c index a2ab31e879a..f343e13d4e9 100644 --- a/lib/libsqlite3/src/vdbe.c +++ b/lib/libsqlite3/src/vdbe.c @@ -152,11 +152,7 @@ int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#ifdef SQLITE_OMIT_MERGE_SORT -# define isSorter(x) 0 -#else # define isSorter(x) ((x)->pSorter!=0) -#endif /* ** Argument pMem points at a register that will be passed to a @@ -869,7 +865,7 @@ case OP_Halt: { if( rc==SQLITE_BUSY ){ p->rc = rc = SQLITE_BUSY; }else{ - assert( rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ); + assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); assert( rc==SQLITE_OK || db->nDeferredCons>0 ); rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; } @@ -3321,17 +3317,12 @@ case OP_OpenEphemeral: { case OP_SorterOpen: { VdbeCursor *pCx; -#ifndef SQLITE_OMIT_MERGE_SORT pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); -#else - pOp->opcode = OP_OpenEphemeral; - pc--; -#endif break; } @@ -3524,7 +3515,7 @@ case OP_SeekGt: { /* jump, in3 */ ** r.flags = 0; ** } */ - r.flags = (u16)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); + r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekGe || r.flags==0 ); @@ -4214,15 +4205,10 @@ case OP_SorterCompare: { case OP_SorterData: { VdbeCursor *pC; -#ifndef SQLITE_OMIT_MERGE_SORT pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); -#else - pOp->opcode = OP_RowKey; - pc--; -#endif break; } @@ -4421,9 +4407,6 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Sort; -#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4502,9 +4485,6 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Next; -#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4555,9 +4535,6 @@ case OP_Next: { /* jump */ ** for tables is OP_Insert. */ case OP_SorterInsert: /* in2 */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_IdxInsert; -#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; @@ -5784,7 +5761,7 @@ case OP_VOpen: { /* Initialize sqlite3_vtab_cursor base class */ pVtabCursor->pVtab = pVtab; - /* Initialise vdbe cursor object */ + /* Initialize vdbe cursor object */ pCur = allocateCursor(p, pOp->p1, 0, -1, 0); if( pCur ){ pCur->pVtabCursor = pVtabCursor; @@ -6063,7 +6040,7 @@ case OP_VUpdate: { assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); db->lastRowid = lastRowid = rowid; } - if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ rc = SQLITE_OK; }else{ diff --git a/lib/libsqlite3/src/vdbeInt.h b/lib/libsqlite3/src/vdbeInt.h index fac0b2f6ed3..3a5b4028bbd 100644 --- a/lib/libsqlite3/src/vdbeInt.h +++ b/lib/libsqlite3/src/vdbeInt.h @@ -19,6 +19,14 @@ #define _VDBEINT_H_ /* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 50 +#endif + +/* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance ** of the following structure. @@ -123,7 +131,7 @@ struct VdbeFrame { VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ - u16 nCursor; /* Number of entries in apCsr */ + int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ int nMem; /* Number of entries in aMem */ @@ -309,7 +317,7 @@ struct Vdbe { int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ u16 nResColumn; /* Number of columns in one row of the result set */ - u16 nCursor; /* Number of slots in apCsr[] */ + int nCursor; /* Number of slots in apCsr[] */ u32 magic; /* Magic number for sanity checking */ char *zErrMsg; /* Error message written here */ Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ @@ -429,15 +437,6 @@ int sqlite3VdbeFrameRestore(VdbeFrame *); void sqlite3VdbeMemStoreType(Mem *pMem); int sqlite3VdbeTransferError(Vdbe *p); -#ifdef SQLITE_OMIT_MERGE_SORT -# define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterWrite(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterClose(Y,Z) -# define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK -#else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); @@ -445,7 +444,6 @@ int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int *); -#endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 void sqlite3VdbeEnter(Vdbe*); diff --git a/lib/libsqlite3/src/vdbeapi.c b/lib/libsqlite3/src/vdbeapi.c index b48826680a3..7c861e2d472 100644 --- a/lib/libsqlite3/src/vdbeapi.c +++ b/lib/libsqlite3/src/vdbeapi.c @@ -445,7 +445,7 @@ end_of_step: assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ /* If this statement was prepared using sqlite3_prepare_v2(), and an - ** error has occured, then return the error code in p->rc to the + ** error has occurred, then return the error code in p->rc to the ** caller. Set the error code in the database handle to the same value. */ rc = sqlite3VdbeTransferError(p); @@ -454,14 +454,6 @@ end_of_step: } /* -** The maximum number of times that a statement will try to reparse -** itself before giving up and returning SQLITE_SCHEMA. -*/ -#ifndef SQLITE_MAX_SCHEMA_RETRY -# define SQLITE_MAX_SCHEMA_RETRY 5 -#endif - -/* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. @@ -1198,7 +1190,7 @@ int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ if( zName ){ for(i=0; i<p->nzVar; i++){ const char *z = p->azVar[i]; - if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ + if( z && strncmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; } } diff --git a/lib/libsqlite3/src/vdbeaux.c b/lib/libsqlite3/src/vdbeaux.c index 3bf8e471482..2c4269a59e6 100644 --- a/lib/libsqlite3/src/vdbeaux.c +++ b/lib/libsqlite3/src/vdbeaux.c @@ -17,18 +17,6 @@ #include "sqliteInt.h" #include "vdbeInt.h" - - -/* -** When debugging the code generator in a symbolic debugger, one can -** set the sqlite3VdbeAddopTrace to 1 and all opcodes will be printed -** as they are added to the instruction stream. -*/ -#ifdef SQLITE_DEBUG -int sqlite3VdbeAddopTrace = 0; -#endif - - /* ** Create a new virtual database engine. */ @@ -158,7 +146,9 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p4type = P4_NOTUSED; #ifdef SQLITE_DEBUG pOp->zComment = 0; - if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + } #endif #ifdef VDBE_PROFILE pOp->cycles = 0; @@ -377,7 +367,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) #endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && (pOp->p1==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; @@ -385,7 +375,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ } sqlite3DbFree(v->db, sIter.apSub); - /* Return true if hasAbort==mayAbort. Or if a malloc failure occured. + /* Return true if hasAbort==mayAbort. Or if a malloc failure occurred. ** If malloc failed, then the while() loop above may not have iterated ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame @@ -512,7 +502,7 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ pOut->p5 = 0; #ifdef SQLITE_DEBUG pOut->zComment = 0; - if( sqlite3VdbeAddopTrace ){ + if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } #endif @@ -1538,7 +1528,7 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); - p->nCursor = (u16)nCursor; + p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ p->nVar = (ynVar)nVar; @@ -1780,7 +1770,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( needXcommit && db->xCommitCallback ){ rc = db->xCommitCallback(db->pCommitArg); if( rc ){ - return SQLITE_CONSTRAINT; + return SQLITE_CONSTRAINT_COMMITHOOK; } } @@ -2017,7 +2007,7 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* If p->iStatement is greater than zero, then this Vdbe opened a ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occured, causing an emergency rollback. + ** is that an IO error may have occurred, causing an emergency rollback. ** In this case (db->nStatement==0), and there is nothing to do. */ if( db->nStatement && p->iStatement ){ @@ -2072,14 +2062,14 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ ** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT and write -** an error message to it. Then return SQLITE_ERROR. +** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. Then return SQLITE_ERROR. */ #ifndef SQLITE_OMIT_FOREIGN_KEY int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ sqlite3 *db = p->db; if( (deferred && db->nDeferredCons>0) || (!deferred && p->nFkConstraint>0) ){ - p->rc = SQLITE_CONSTRAINT; + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; sqlite3SetString(&p->zErrMsg, db, "foreign key constraint failed"); return SQLITE_ERROR; @@ -2153,7 +2143,7 @@ int sqlite3VdbeHalt(Vdbe *p){ ** ** Even if the statement is read-only, it is important to perform ** a statement or transaction rollback operation. If the error - ** occured while writing to the journal, sub-journal or database + ** occurred while writing to the journal, sub-journal or database ** file as part of an effort to free up cache space (see function ** pagerStress() in pager.c), the rollback is required to restore ** the pager to a consistent state. @@ -2194,7 +2184,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeLeave(p); return SQLITE_ERROR; } - rc = SQLITE_CONSTRAINT; + rc = SQLITE_CONSTRAINT_FOREIGNKEY; }else{ /* The auto-commit flag is true, the vdbe program was successful ** or hit an 'OR FAIL' constraint and there are no deferred foreign @@ -2237,7 +2227,7 @@ int sqlite3VdbeHalt(Vdbe *p){ if( eStatementOp ){ rc = sqlite3VdbeCloseStatement(p, eStatementOp); if( rc ){ - if( p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ){ + if( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ){ p->rc = rc; sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -2478,7 +2468,7 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); #if defined(SQLITE_ENABLE_TREE_EXPLAIN) - sqlite3_free(p->zExplain); + sqlite3DbFree(db, p->zExplain); sqlite3DbFree(db, p->pExplain); #endif } @@ -2567,7 +2557,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ ** the blob of data that it corresponds to. In a table record, all serial ** types are stored at the start of the record, and the blobs of data at ** the end. Hence these functions allow the caller to handle the -** serial-type and data blob seperately. +** serial-type and data blob separately. ** ** The following table describes the various storage classes for data: ** diff --git a/lib/libsqlite3/src/vdbeblob.c b/lib/libsqlite3/src/vdbeblob.c index ae77a47ba36..2e8fd8ee744 100644 --- a/lib/libsqlite3/src/vdbeblob.c +++ b/lib/libsqlite3/src/vdbeblob.c @@ -313,7 +313,7 @@ int sqlite3_blob_open( } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); + } while( (++nAttempt)<SQLITE_MAX_SCHEMA_RETRY && rc==SQLITE_SCHEMA ); blob_open_out: if( rc==SQLITE_OK && db->mallocFailed==0 ){ diff --git a/lib/libsqlite3/src/vdbemem.c b/lib/libsqlite3/src/vdbemem.c index fd964de2e91..8fc222e2de2 100644 --- a/lib/libsqlite3/src/vdbemem.c +++ b/lib/libsqlite3/src/vdbemem.c @@ -32,7 +32,9 @@ ** between formats. */ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ +#ifndef SQLITE_OMIT_UTF16 int rc; +#endif assert( (pMem->flags&MEM_RowSet)==0 ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); diff --git a/lib/libsqlite3/src/vdbesort.c b/lib/libsqlite3/src/vdbesort.c index d51bbf54a3e..fdfc4a79ddc 100644 --- a/lib/libsqlite3/src/vdbesort.c +++ b/lib/libsqlite3/src/vdbesort.c @@ -18,7 +18,6 @@ #include "sqliteInt.h" #include "vdbeInt.h" -#ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SorterRecord SorterRecord; @@ -1037,5 +1036,3 @@ int sqlite3VdbeSorterCompare( vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } - -#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ diff --git a/lib/libsqlite3/src/vdbetrace.c b/lib/libsqlite3/src/vdbetrace.c index 35825c8736e..356277e8d23 100644 --- a/lib/libsqlite3/src/vdbetrace.c +++ b/lib/libsqlite3/src/vdbetrace.c @@ -53,6 +53,11 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** ** The calling function is responsible for making sure the memory returned ** is eventually freed. ** @@ -123,30 +128,49 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ + int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ - Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else + pVar = &utf8; + } #endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ){ + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOut<pVar->n && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } +#endif + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut<pVar->n ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ + int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; i<pVar->n; i++){ + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; i<nOut; i++){ sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut<pVar->n ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); +#endif } } } diff --git a/lib/libsqlite3/src/wal.c b/lib/libsqlite3/src/wal.c index 0d7271bf04f..e642ea2105b 100644 --- a/lib/libsqlite3/src/wal.c +++ b/lib/libsqlite3/src/wal.c @@ -1207,8 +1207,9 @@ finished: ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName ); } } @@ -1722,8 +1723,8 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); @@ -1733,6 +1734,7 @@ static int walCheckpoint( } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -2287,19 +2289,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2315,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2386,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). @@ -2952,6 +2957,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ diff --git a/lib/libsqlite3/src/wal.h b/lib/libsqlite3/src/wal.h index a848de15276..092546354b3 100644 --- a/lib/libsqlite3/src/wal.h +++ b/lib/libsqlite3/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -44,6 +43,7 @@ # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); diff --git a/lib/libsqlite3/src/walker.c b/lib/libsqlite3/src/walker.c index eab96ea24dc..e71ed2ac484 100644 --- a/lib/libsqlite3/src/walker.c +++ b/lib/libsqlite3/src/walker.c @@ -113,7 +113,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. Invoke the xSelectCallback() +** either before or after the walk of expressions and FROM clause, depending +** on whether pWalker->bSelectDepthFirst is false or true, respectively. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -127,14 +129,23 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){ rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; + if( !pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } + if( pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + /* Depth-first search is currently only used for + ** selectAddSubqueryTypeInfo() and that routine always returns + ** WRC_Continue (0). So the following branch is never taken. */ + if( NEVER(rc) ) break; + } p = p->pPrior; } pWalker->walkerDepth--; diff --git a/lib/libsqlite3/src/where.c b/lib/libsqlite3/src/where.c index fb3bc5c93cf..e614f4a6d86 100644 --- a/lib/libsqlite3/src/where.c +++ b/lib/libsqlite3/src/where.c @@ -98,8 +98,8 @@ struct WhereTerm { int leftCursor; /* Cursor number of X in "X <op> <expr>" */ union { int leftColumn; /* Column number of X in "X <op> <expr>" */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; u16 eOperator; /* A WO_xx value describing <op> */ u8 wtFlags; /* TERM_xxx bit flags. See below */ @@ -140,7 +140,6 @@ struct WhereTerm { struct WhereClause { Parse *pParse; /* The parser context */ WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ @@ -227,6 +226,7 @@ struct WhereCost { #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ @@ -253,7 +253,7 @@ struct WhereCost { #define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ #define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ #define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ +#define WHERE_IN_ABLE 0x080f1000 /* Able to support an IN operator */ #define WHERE_TOP_LIMIT 0x00100000 /* x<EXPR or x<=EXPR constraint */ #define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and x<EXPR */ @@ -262,6 +262,8 @@ struct WhereCost { #define WHERE_REVERSE 0x01000000 /* Scan in reverse order */ #define WHERE_UNIQUE 0x02000000 /* Selects no more than one row */ #define WHERE_ALL_UNIQUE 0x04000000 /* This and all prior have one row */ +#define WHERE_OB_UNIQUE 0x00004000 /* Values in ORDER BY columns are + ** different for every output row */ #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ @@ -316,7 +318,6 @@ static void whereClauseInit( pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; pWC->wctrlFlags = wctrlFlags; } @@ -563,7 +564,7 @@ static int allowedOp(int op){ ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If left/right precendence rules come into play when determining the +** If left/right precedence rules come into play when determining the ** collating ** side of the comparison, it remains associated with the same side after ** the commutation. So "Y collate NOCASE op X" becomes @@ -629,6 +630,23 @@ static u16 operatorMask(int op){ ** where X is a reference to the iColumn of table iCur and <op> is one of ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y=<expr> if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return <expr> if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=<expr>. +** +** If there are multiple terms in the WHERE clause of the form "X <op> <expr>" +** then try for the one with no dependencies on <expr> - in other words where +** <expr> is a constant expression of some kind. Only return entries of +** the form "X <op> Y" where Y is a column in another table if no terms of +** the form "X <op> <const-expr>" exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ static WhereTerm *findTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -638,45 +656,85 @@ static WhereTerm *findTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - WhereTerm *pTerm; - int k; + WhereTerm *pTerm; /* Term being examined as possible result */ + WhereTerm *pResult = 0; /* The answer to return */ + WhereClause *pWCOrig = pWC; /* Original pWC value */ + int j, k; /* Loop counters */ + Expr *pX; /* Pointer to an expression */ + Parse *pParse; /* Parsing context */ + int iOrigCol = iColumn; /* Original value of iColumn */ + int nEquiv = 2; /* Number of entires in aEquiv[] */ + int iEquiv = 2; /* Number of entries of aEquiv[] processed so far */ + int aEquiv[22]; /* iCur,iColumn and up to 10 other equivalents */ + assert( iCur>=0 ); - op &= WO_ALL; - for(; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; + aEquiv[0] = iCur; + aEquiv[1] = iColumn; + for(;;){ + for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){ + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + ){ + if( (pTerm->prereqRight & notReady)==0 + && (pTerm->eOperator & op & WO_ALL)!=0 + ){ + if( iOrigCol>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + char idxaff; + + pX = pTerm->pExpr; + pParse = pWC->pParse; + idxaff = pIdx->pTable->aCol[iOrigCol].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ){ + continue; + } + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + + for(j=0; pIdx->aiColumn[j]!=iOrigCol; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ){ + continue; + } + } + if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ + pResult = pTerm; + goto findTerm_success; + }else if( pResult==0 ){ + pResult = pTerm; + } + } + if( (pTerm->eOperator & WO_EQUIV)!=0 + && nEquiv<ArraySize(aEquiv) + ){ + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; j<nEquiv; j+=2){ + if( aEquiv[j]==pX->iTable && aEquiv[j+1]==pX->iColumn ) break; + } + if( j==nEquiv ){ + aEquiv[j] = pX->iTable; + aEquiv[j+1] = pX->iColumn; + nEquiv += 2; + } } - if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; } - return pTerm; } } + if( iEquiv>=nEquiv ) break; + iCur = aEquiv[iEquiv++]; + iColumn = aEquiv[iEquiv++]; } - return 0; +findTerm_success: + return pResult; } /* Forward reference */ @@ -862,7 +920,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: @@ -950,11 +1008,10 @@ static void exprAnalyzeOrTerm( ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); @@ -993,7 +1050,7 @@ static void exprAnalyzeOrTerm( b |= getMask(pMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; @@ -1044,7 +1101,7 @@ static void exprAnalyzeOrTerm( for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and @@ -1070,7 +1127,7 @@ static void exprAnalyzeOrTerm( /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); + assert( IsPowerOfTwo(chngToIN) ); assert( chngToIN==getMask(pMaskSet, iCursor) ); break; } @@ -1080,7 +1137,7 @@ static void exprAnalyzeOrTerm( ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ @@ -1116,7 +1173,7 @@ static void exprAnalyzeOrTerm( for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); @@ -1146,7 +1203,6 @@ static void exprAnalyzeOrTerm( } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -1215,17 +1271,19 @@ static void exprAnalyze( pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ + if( allowedOp(op) ){ Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pLeft->op==TK_COLUMN ){ pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + pTerm->eOperator = operatorMask(op) & opMask; } if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -1240,6 +1298,13 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; @@ -1251,7 +1316,7 @@ static void exprAnalyze( testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } } @@ -1710,7 +1775,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ - if( pTerm->eOperator==WO_OR + if( (pTerm->eOperator & WO_OR)!=0 && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){ @@ -1731,7 +1796,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", (pOrTerm - pOrWC->a), (pTerm - pWC->a) )); - if( pOrTerm->eOperator==WO_AND ){ + if( (pOrTerm->eOperator& WO_AND)!=0 ){ sBOI.pWC = &pOrTerm->u.pAndInfo->wc; bestIndex(&sBOI); }else if( pOrTerm->leftCursor==iCur ){ @@ -1792,7 +1857,7 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; @@ -2054,10 +2119,10 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } @@ -2105,15 +2170,18 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ pUsage; for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ + u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The ** following asserts verify this fact. */ @@ -2123,7 +2191,7 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); j++; } for(i=0; i<nOrderBy; i++){ @@ -2209,6 +2277,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ WhereTerm *pTerm; int i, j; int nOrderBy; + int bAllowIN; /* Allow IN optimizations */ double rCost; /* Make sure wsFlags is initialized to some sane value. Otherwise, if the @@ -2243,59 +2312,106 @@ static void bestVirtualIndex(WhereBestIdx *p){ assert( pTab->azModuleArg && pTab->azModuleArg[0] ); assert( sqlite3GetVTable(pParse->db, pTab) ); - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. + /* Try once or twice. On the first attempt, allow IN optimizations. + ** If an IN optimization is accepted by the virtual table xBestIndex + ** method, but the pInfo->aConstrainUsage.omit flag is not set, then + ** the query will not work because it might allow duplicate rows in + ** output. In that case, run the xBestIndex method a second time + ** without the IN constraints. Usually this loop only runs once. + ** The loop will exit using a "break" statement. */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight&p->notReady) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !p->pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } + for(bAllowIN=1; 1; bAllowIN--){ + assert( bAllowIN==0 || bAllowIN==1 ); - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - p->cost.used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; + /* Set the aConstraint[].usable fields and initialize all + ** output variables to zero. + ** + ** aConstraint[].usable is true for constraints where the right-hand + ** side contains only references to tables to the left of the current + ** table. In other words, if the constraint is of the form: + ** + ** column = expr + ** + ** and we are evaluating a join, then the constraint on column is + ** only valid if all tables referenced in expr occur to the left + ** of the table containing column. + ** + ** The aConstraints[] array contains entries for all constraints + ** on the current table. That way we only have to compute it once + ** even though we might try to pick the best index multiple times. + ** For each attempt at picking an index, the order of tables in the + ** join might be different so we have to recompute the usable flag + ** each time. + */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pUsage = pIdxInfo->aConstraintUsage; + for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + if( (pTerm->prereqRight&p->notReady)==0 + && (bAllowIN || (pTerm->eOperator & WO_IN)==0) + ){ + pIdxCons->usable = 1; + }else{ + pIdxCons->usable = 0; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + } + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + nOrderBy = pIdxInfo->nOrderBy; + if( !p->pOrderBy ){ + pIdxInfo->nOrderBy = 0; } + + if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ + return; + } + + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){ + if( pUsage[i].argvIndex>0 ){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + p->cost.used |= pTerm->prereqRight; + if( (pTerm->eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=pIdxInfo->nConstraint ) break; } + /* The orderByConsumed signal is only valid if all outer loops collectively + ** generate just a single row of output. + */ + if( pIdxInfo->orderByConsumed ){ + for(i=0; i<p->i; i++){ + if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ + pIdxInfo->orderByConsumed = 0; + } + } + } + /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This ** matches the processing for non-virtual tables in bestBtreeIndex(). @@ -2590,24 +2706,24 @@ static int whereRangeScanEst( if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK ){ iLower = a[0]; - if( pLower->eOperator==WO_GT ) iLower += a[1]; + if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK ){ iUpper = a[0]; - if( pUpper->eOperator==WO_LE ) iUpper += a[1]; + if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } sqlite3ValueFree(pRangeVal); } @@ -2797,7 +2913,8 @@ static int isSortingIndex( WhereBestIdx *p, /* Best index search context */ Index *pIdx, /* The index we are testing */ int base, /* Cursor number for the table to be sorted */ - int *pbRev /* Set to 1 for reverse-order scan of pIdx */ + int *pbRev, /* Set to 1 for reverse-order scan of pIdx */ + int *pbObUnique /* ORDER BY column values will different in every row */ ){ int i; /* Number of pIdx terms used */ int j; /* Number of ORDER BY terms satisfied */ @@ -2811,12 +2928,16 @@ static int isSortingIndex( int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ + int outerObUnique; /* Outer loops generate different values in + ** every row for the ORDER BY columns */ if( p->i==0 ){ nPriorSat = 0; + outerObUnique = 1; }else{ + u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags; nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - if( (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ + if( (wsFlags & WHERE_ORDERED)==0 ){ /* This loop cannot be ordered unless the next outer loop is ** also ordered */ return nPriorSat; @@ -2826,6 +2947,9 @@ static int isSortingIndex( ** optimization is disabled */ return nPriorSat; } + testcase( wsFlags & WHERE_OB_UNIQUE ); + testcase( wsFlags & WHERE_ALL_UNIQUE ); + outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0; } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); @@ -2915,12 +3039,9 @@ static int isSortingIndex( WO_EQ|WO_ISNULL|WO_IN, pIdx); if( pConstraint==0 ){ isEq = 0; - }else if( pConstraint->eOperator==WO_IN ){ - /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY - ** because we do not know in what order the values on the RHS of the IN - ** operator will occur. */ - break; - }else if( pConstraint->eOperator==WO_ISNULL ){ + }else if( (pConstraint->eOperator & WO_IN)!=0 ){ + isEq = 0; + }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ uniqueNotNull = 0; isEq = 1; /* "X IS NULL" means X has only a single value */ }else if( pConstraint->prereqRight==0 ){ @@ -2970,11 +3091,26 @@ static int isSortingIndex( uniqueNotNull = 0; } } + if( seenRowid ){ + uniqueNotNull = 1; + }else if( uniqueNotNull==0 || i<pIdx->nColumn ){ + uniqueNotNull = 0; + } /* If we have not found at least one ORDER BY term that matches the ** index, then show no progress. */ if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; + /* Either the outer queries must generate rows where there are no two + ** rows with the same values in all ORDER BY columns, or else this + ** loop must generate just a single row of output. Example: Suppose + ** the outer loops generate A=1 and A=1, and this loop generates B=3 + ** and B=4. Then without the following test, ORDER BY A,B would + ** generate the wrong order output: 1,3 1,4 1,3 1,4 + */ + if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat; + *pbObUnique = uniqueNotNull; + /* Return the necessary scan order back to the caller */ *pbRev = sortOrder & 1; @@ -2982,7 +3118,7 @@ static int isSortingIndex( ** possible for a single row from this table to match, then skip over ** any additional ORDER BY terms dealing with this table. */ - if( seenRowid || (uniqueNotNull && i>=pIdx->nColumn) ){ + if( uniqueNotNull ){ /* Advance j over additional ORDER BY terms associated with base */ WhereMaskSet *pMS = p->pWC->pMaskSet; Bitmask m = ~getMask(pMS, base); @@ -3223,8 +3359,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** indicate this to the caller. ** ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (pc.plan.nEq+1) that can be - ** optimized using the index. + ** there is a range constraint on indexed column (pc.plan.nEq+1) that + ** can be optimized using the index. */ if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); @@ -3266,12 +3402,14 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** variable. */ if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ int bRev = 2; - WHERETRACE((" --> before isSortingIndex: nPriorSat=%d\n",nPriorSat)); - pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev); - WHERETRACE((" --> after isSortingIndex: bRev=%d nOBSat=%d\n", - bRev, pc.plan.nOBSat)); + int bObUnique = 0; + WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat)); + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); + WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", + bRev, bObUnique, pc.plan.nOBSat)); if( nPriorSat<pc.plan.nOBSat || (pc.plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ pc.plan.wsFlags |= WHERE_ORDERED; + if( bObUnique ) pc.plan.wsFlags |= WHERE_OB_UNIQUE; } if( nOrderBy==pc.plan.nOBSat ){ bSort = 0; @@ -3333,12 +3471,13 @@ static void bestBtreeIndex(WhereBestIdx *p){ && pFirstTerm!=0 && aiRowEst[1]>1 ){ assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); + testcase( pFirstTerm->eOperator & WO_EQ ); + testcase( pFirstTerm->eOperator & WO_EQUIV ); + testcase( pFirstTerm->eOperator & WO_ISNULL ); whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &pc.plan.nRow); }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator==WO_IN ); + assert( pFirstTerm->eOperator & WO_IN ); whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &pc.plan.nRow); } @@ -3364,7 +3503,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** So this computation assumes table records are about twice as big ** as index records */ - if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED))==WHERE_IDX_ONLY + if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE)) + ==WHERE_IDX_ONLY && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) @@ -3485,7 +3625,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** selective in practice, on average. */ pc.plan.nRow /= 3; } - }else if( pTerm->eOperator!=WO_NOOP ){ + }else if( (pTerm->eOperator & WO_NOOP)==0 ){ /* Any other expression lowers the output row count by half */ pc.plan.nRow /= 2; } @@ -3524,7 +3664,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag ** is set, then reverse the order that the index will be scanned ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that + ** where application behavior depends on the (undefined) order that ** SQLite outputs rows in in the absence of an ORDER BY clause. */ if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ p->cost.plan.wsFlags |= WHERE_REVERSE; @@ -3537,8 +3677,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ || p->cost.plan.u.pIdx==pSrc->pIndex ); - WHERETRACE((" best index is: %s\n", - p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk")); + WHERETRACE((" best index is %s cost=%.1f\n", + p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk", + p->cost.rCost)); bestOrClauseIndex(p); bestAutomaticIndex(p); @@ -3563,7 +3704,8 @@ static void bestIndex(WhereBestIdx *p){ sqlite3_index_info *pIdxInfo = 0; p->ppIdxInfo = &pIdxInfo; bestVirtualIndex(p); - if( pIdxInfo->needToFreeIdxStr ){ + assert( pIdxInfo!=0 || p->pParse->db->mallocFailed ); + if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){ sqlite3_free(pIdxInfo->idxStr); } sqlite3DbFree(p->pParse->db, pIdxInfo); @@ -3669,7 +3811,8 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ static int codeEqualityTerm( Parse *pParse, /* The parsing context */ WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; @@ -3687,12 +3830,26 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; + u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 + && pLevel->plan.u.pIdx->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 ); + testcase( iEq>0 && iEq+1<pLevel->plan.u.pIdx->nColumn ); + testcase( bRev ); + bRev = !bRev; + } assert( pX->op==TK_IN ); iReg = iTarget; eType = sqlite3FindInIndex(pParse, pX, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); @@ -3710,6 +3867,7 @@ static int codeEqualityTerm( }else{ pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; sqlite3VdbeAddOp1(v, OP_IsNull, iReg); }else{ pLevel->u.in.nIn = 0; @@ -3804,7 +3962,7 @@ static int codeAllEqualityTerms( ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); @@ -4014,6 +4172,7 @@ static Bitmask codeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -4024,6 +4183,7 @@ static Bitmask codeOneLoopStart( bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -4064,6 +4224,7 @@ static Bitmask codeOneLoopStart( ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; int nConstraint = pVtabIdx->nConstraint; struct sqlite3_index_constraint_usage *aUsage = @@ -4073,11 +4234,18 @@ static Bitmask codeOneLoopStart( sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; for(j=1; j<=nConstraint; j++){ for(k=0; k<nConstraint; k++){ if( aUsage[k].argvIndex==j ){ - int iTerm = aConstraint[k].iTermOffset; - sqlite3ExprCode(pParse, pWC->a[iTerm].pExpr->pRight, iReg+j+1); + int iTarget = iReg+j+1; + pTerm = &pWC->a[aConstraint[k].iTermOffset]; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, k, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } break; } } @@ -4085,7 +4253,7 @@ static Bitmask codeOneLoopStart( } sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pVtabIdx->idxStr, pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); pVtabIdx->needToFreeIdxStr = 0; for(j=0; j<nConstraint; j++){ @@ -4112,13 +4280,13 @@ static Bitmask codeOneLoopStart( pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); pLevel->op = OP_Noop; @@ -4503,7 +4671,7 @@ static Bitmask codeOneLoopStart( pTerm = pLevel->plan.u.pTerm; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; @@ -4558,6 +4726,10 @@ static Bitmask codeOneLoopStart( ** the "interesting" terms of z - terms that did not originate in the ** ON or USING clause of a LEFT JOIN, and terms that are usable as ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 */ if( pWC->nTerm>1 ){ int iTerm; @@ -4576,10 +4748,10 @@ static Bitmask codeOneLoopStart( for(ii=0; ii<pOrWc->nTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; - if( pAndExpr ){ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } @@ -4666,7 +4838,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -4680,7 +4852,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -4695,6 +4867,33 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -4707,7 +4906,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -4718,7 +4917,7 @@ static Bitmask codeOneLoopStart( } sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return newNotReady; } #if defined(SQLITE_TEST) @@ -4954,24 +5153,13 @@ WhereInfo *sqlite3WhereBegin( ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 ); for(ii=0; ii<pTabList->nSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[ii].pTab) && IsVirtual(pTabList->a[ii].pTab) ){ - sWBI.pWC->vmask |= ((Bitmask)1 << ii); - } -#endif } #ifndef NDEBUG { @@ -5031,6 +5219,7 @@ WhereInfo *sqlite3WhereBegin( int bestJ = -1; /* The value of j */ Bitmask m; /* Bitmask value for j or bestJ */ int isOptimal; /* Iterator for optimal/non-optimal search */ + int ckOptimal; /* Do the optimal scan check */ int nUnconstrained; /* Number tables without INDEXED BY */ Bitmask notIndexed; /* Mask of tables that cannot use an index */ @@ -5065,10 +5254,8 @@ WhereInfo *sqlite3WhereBegin( ** strategies were found by the first iteration. This second iteration ** is used to search for the lowest cost scan overall. ** - ** Previous versions of SQLite performed only the second iteration - - ** the next outermost loop was always that with the lowest overall - ** cost. However, this meant that SQLite could select the wrong plan - ** for scripts such as the following: + ** Without the optimal scan step (the first iteration) a suboptimal + ** plan might be chosen for queries like this: ** ** CREATE TABLE t1(a, b); ** CREATE TABLE t2(c, d); @@ -5083,17 +5270,41 @@ WhereInfo *sqlite3WhereBegin( */ nUnconstrained = 0; notIndexed = 0; - for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){ + + /* The optimal scan check only occurs if there are two or more tables + ** available to be reordered */ + if( iFrom==nTabList-1 ){ + ckOptimal = 0; /* Common case of just one table in the FROM clause */ + }else{ + ckOptimal = -1; for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; j<nTabList; j++, sWBI.pSrc++){ - int doNotReorder; /* True if this table should not be reordered */ - - doNotReorder = (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; m = getMask(pMaskSet, sWBI.pSrc->iCursor); if( (m & sWBI.notValid)==0 ){ if( j==iFrom ) iFrom++; continue; } + if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ) break; + if( ++ckOptimal ) break; + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; + } + } + assert( ckOptimal==0 || ckOptimal==1 ); + + for(isOptimal=ckOptimal; isOptimal>=0 && bestJ<0; isOptimal--){ + for(j=iFrom, sWBI.pSrc=&pTabList->a[j]; j<nTabList; j++, sWBI.pSrc++){ + if( j>iFrom && (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0 ){ + /* This break and one like it in the ckOptimal computation loop + ** above prevent table reordering across LEFT and CROSS JOINs. + ** The LEFT JOIN case is necessary for correctness. The prohibition + ** against reordering across a CROSS JOIN is an SQLite feature that + ** allows the developer to control table reordering */ + break; + } + m = getMask(pMaskSet, sWBI.pSrc->iCursor); + if( (m & sWBI.notValid)==0 ){ + assert( j>iFrom ); + continue; + } sWBI.notReady = (isOptimal ? m : sWBI.notValid); if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; @@ -5122,8 +5333,8 @@ WhereInfo *sqlite3WhereBegin( } if( isOptimal ){ pWInfo->a[j].rOptCost = sWBI.cost.rCost; - }else if( iFrom<nTabList-1 ){ - /* If two or more tables have nearly the same outer loop cost, + }else if( ckOptimal ){ + /* If two or more tables have nearly the same outer loop cost, but ** very different inner loop (optimal) cost, we want to choose ** for the outer loop that table which benefits the least from ** being in the inner loop. The following code scales the @@ -5168,11 +5379,19 @@ WhereInfo *sqlite3WhereBegin( bestPlan = sWBI.cost; bestJ = j; } - if( doNotReorder ) break; + + /* In a join like "w JOIN x LEFT JOIN y JOIN z" make sure that + ** table y (and not table z) is always the next inner loop inside + ** of table x. */ + if( (sWBI.pSrc->jointype & JT_LEFT)!=0 ) break; } } assert( bestJ>=0 ); assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); + assert( bestJ==iFrom || (pTabList->a[iFrom].jointype & JT_LEFT)==0 ); + testcase( bestJ>iFrom && (pTabList->a[iFrom].jointype & JT_CROSS)!=0 ); + testcase( bestJ>iFrom && bestJ<nTabList-1 + && (pTabList->a[bestJ+1].jointype & JT_LEFT)!=0 ); WHERETRACE(("*** Optimizer selects table %d (%s) for loop %d with:\n" " cost=%.1f, nRow=%.1f, nOBSat=%d, wsFlags=0x%08x\n", bestJ, pTabList->a[bestJ].pTab->zName, @@ -5424,7 +5643,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); |