summaryrefslogtreecommitdiffstats
path: root/lib/libsqlite3/ext
diff options
context:
space:
mode:
Diffstat (limited to 'lib/libsqlite3/ext')
-rw-r--r--lib/libsqlite3/ext/fts3/fts3.c154
-rw-r--r--lib/libsqlite3/ext/fts3/fts3Int.h18
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_aux.c2
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_expr.c4
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_icu.c13
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_snippet.c277
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_term.c2
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_tokenize_vtab.c2
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_tokenizer.c10
-rw-r--r--lib/libsqlite3/ext/fts3/fts3_write.c13
-rw-r--r--lib/libsqlite3/ext/fts3/unicode/mkunicode.tcl219
-rw-r--r--lib/libsqlite3/ext/fts3/unicode/parseunicode.tcl146
-rw-r--r--lib/libsqlite3/ext/icu/icu.c4
-rw-r--r--lib/libsqlite3/ext/misc/fuzzer.c2
-rw-r--r--lib/libsqlite3/ext/misc/spellfix.c38
-rw-r--r--lib/libsqlite3/ext/rtree/rtree.c64
-rw-r--r--lib/libsqlite3/ext/rtree/rtree9.test1
-rw-r--r--lib/libsqlite3/ext/rtree/rtreeC.test83
-rw-r--r--lib/libsqlite3/ext/rtree/rtreeE.test15
-rw-r--r--lib/libsqlite3/ext/rtree/sqlite3rtree.h2
20 files changed, 758 insertions, 311 deletions
diff --git a/lib/libsqlite3/ext/fts3/fts3.c b/lib/libsqlite3/ext/fts3/fts3.c
index 7a15379b83e..6a9b507fc0d 100644
--- a/lib/libsqlite3/ext/fts3/fts3.c
+++ b/lib/libsqlite3/ext/fts3/fts3.c
@@ -313,6 +313,13 @@ static int fts3EvalStart(Fts3Cursor *pCsr);
static int fts3TermSegReaderCursor(
Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
+#ifndef SQLITE_AMALGAMATION
+# if defined(SQLITE_DEBUG)
+int sqlite3Fts3Always(int b) { assert( b ); return b; }
+int sqlite3Fts3Never(int b) { assert( !b ); return b; }
+# endif
+#endif
+
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
@@ -422,7 +429,7 @@ void sqlite3Fts3Dequote(char *z){
/* If the first byte was a '[', then the close-quote character is a ']' */
if( quote=='[' ) quote = ']';
- while( ALWAYS(z[iIn]) ){
+ while( z[iIn] ){
if( z[iIn]==quote ){
if( z[iIn+1]!=quote ) break;
z[iOut++] = quote;
@@ -502,6 +509,17 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
}
/*
+** Write an error message into *pzErr
+*/
+void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){
+ va_list ap;
+ sqlite3_free(*pzErr);
+ va_start(ap, zFormat);
+ *pzErr = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+}
+
+/*
** Construct one or more SQL statements from the format string given
** and then evaluate those statements. The success code is written
** into *pRc.
@@ -1019,7 +1037,8 @@ static int fts3ContentColumns(
const char *zTbl, /* Name of content table */
const char ***pazCol, /* OUT: Malloc'd array of column names */
int *pnCol, /* OUT: Size of array *pazCol */
- int *pnStr /* OUT: Bytes of string content */
+ int *pnStr, /* OUT: Bytes of string content */
+ char **pzErr /* OUT: error message */
){
int rc = SQLITE_OK; /* Return code */
char *zSql; /* "SELECT *" statement on zTbl */
@@ -1030,6 +1049,9 @@ static int fts3ContentColumns(
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db));
+ }
}
sqlite3_free(zSql);
@@ -1196,13 +1218,13 @@ static int fts3InitVtab(
}
}
if( iOpt==SizeofArray(aFts4Opt) ){
- *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
rc = SQLITE_ERROR;
}else{
switch( iOpt ){
case 0: /* MATCHINFO */
if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
- *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
rc = SQLITE_ERROR;
}
bNoDocsize = 1;
@@ -1230,7 +1252,7 @@ static int fts3InitVtab(
if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
&& (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4))
){
- *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
rc = SQLITE_ERROR;
}
bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
@@ -1281,7 +1303,7 @@ static int fts3InitVtab(
if( nCol==0 ){
sqlite3_free((void*)aCol);
aCol = 0;
- rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
+ rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr);
/* If a languageid= option was specified, remove the language id
** column from the aCol[] array. */
@@ -1316,7 +1338,7 @@ static int fts3InitVtab(
rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
if( rc==SQLITE_ERROR ){
assert( zPrefix );
- *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
+ sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix);
}
if( rc!=SQLITE_OK ) goto fts3_init_out;
@@ -1398,7 +1420,7 @@ static int fts3InitVtab(
}
for(i=0; i<nNotindexed; i++){
if( azNotindexed[i] ){
- *pzErr = sqlite3_mprintf("no such column: %s", azNotindexed[i]);
+ sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotindexed[i]);
rc = SQLITE_ERROR;
}
}
@@ -1406,7 +1428,7 @@ static int fts3InitVtab(
if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){
char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
rc = SQLITE_ERROR;
- *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss);
+ sqlite3Fts3ErrMsg(pzErr, "missing %s parameter in fts4 constructor", zMiss);
}
p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
@@ -1653,7 +1675,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
- sqlite3_free(pCsr->aMatchinfo);
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_free(pCsr);
return SQLITE_OK;
@@ -2799,7 +2821,7 @@ static int fts3SegReaderCursor(
** calls out here. */
if( iLevel<0 && p->aIndex ){
Fts3SegReader *pSeg = 0;
- rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg);
if( rc==SQLITE_OK && pSeg ){
rc = fts3SegReaderCursorAppend(pCsr, pSeg);
}
@@ -3154,7 +3176,7 @@ static int fts3FilterMethod(
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr->aDoclist);
- sqlite3_free(pCsr->aMatchinfo);
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
@@ -3448,11 +3470,31 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){
char *p = &(*ppPoslist)[-2];
char c = 0;
+ /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */
while( p>pStart && (c=*p--)==0 );
+
+ /* Search backwards for a varint with value zero (the end of the previous
+ ** poslist). This is an 0x00 byte preceded by some byte that does not
+ ** have the 0x80 bit set. */
while( p>pStart && (*p & 0x80) | c ){
c = *p--;
}
- if( p>pStart ){ p = &p[2]; }
+ assert( p==pStart || c==0 );
+
+ /* At this point p points to that preceding byte without the 0x80 bit
+ ** set. So to find the start of the poslist, skip forward 2 bytes then
+ ** over a varint.
+ **
+ ** Normally. The other case is that p==pStart and the poslist to return
+ ** is the first in the doclist. In this case do not skip forward 2 bytes.
+ ** The second part of the if condition (c==0 && *ppPoslist>&p[2])
+ ** is required for cases where the first byte of a doclist and the
+ ** doclist is empty. For example, if the first docid is 10, a doclist
+ ** that begins with:
+ **
+ ** 0x0A 0x00 <next docid delta varint>
+ */
+ if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; }
while( *p++&0x80 );
*ppPoslist = p;
}
@@ -3523,6 +3565,8 @@ static void fts3SnippetFunc(
}
if( !zEllipsis || !zEnd || !zStart ){
sqlite3_result_error_nomem(pContext);
+ }else if( nToken==0 ){
+ sqlite3_result_text(pContext, "", -1, SQLITE_STATIC);
}else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken);
}
@@ -4187,7 +4231,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
int bIncrOk = (bOptOk
&& pCsr->bDesc==pTab->bDescIdx
&& p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
- && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
#ifdef SQLITE_TEST
&& pTab->bNoIncrDoclist==0
#endif
@@ -4307,6 +4350,7 @@ void sqlite3Fts3DoclistNext(
p += sqlite3Fts3GetVarint(p, piDocid);
}else{
fts3PoslistCopy(0, &p);
+ while( p<&aDoclist[nDoclist] && *p==0 ) p++;
if( p>=&aDoclist[nDoclist] ){
*pbEof = 1;
}else{
@@ -4584,12 +4628,14 @@ static void fts3EvalStartReaders(
){
if( pExpr && SQLITE_OK==*pRc ){
if( pExpr->eType==FTSQUERY_PHRASE ){
- int i;
int nToken = pExpr->pPhrase->nToken;
- for(i=0; i<nToken; i++){
- if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
+ if( nToken ){
+ int i;
+ for(i=0; i<nToken; i++){
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
+ }
+ pExpr->bDeferred = (i==nToken);
}
- pExpr->bDeferred = (i==nToken);
*pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase);
}else{
fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc);
@@ -5028,7 +5074,7 @@ static int fts3EvalNearTrim(
** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
** advanced to point to the next row that matches "x AND y".
**
-** See fts3EvalTestDeferredAndNear() for details on testing if a row is
+** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is
** really a match, taking into account deferred tokens and NEAR operators.
*/
static void fts3EvalNextRow(
@@ -5248,7 +5294,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
}
/*
-** This function is a helper function for fts3EvalTestDeferredAndNear().
+** This function is a helper function for sqlite3Fts3EvalTestDeferred().
** Assuming no error occurs or has occurred, It returns non-zero if the
** expression passed as the second argument matches the row that pCsr
** currently points to, or zero if it does not.
@@ -5369,7 +5415,7 @@ static int fts3EvalTestExpr(
** Or, if no error occurs and it seems the current row does match the FTS
** query, return 0.
*/
-static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
+int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){
int rc = *pRc;
int bMiss = 0;
if( rc==SQLITE_OK ){
@@ -5416,7 +5462,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){
pCsr->isRequireSeek = 1;
pCsr->isMatchinfoNeeded = 1;
pCsr->iPrevId = pExpr->iDocid;
- }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
+ }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) );
}
/* Check if the cursor is past the end of the docid range specified
@@ -5577,7 +5623,7 @@ static int fts3EvalGatherStats(
pCsr->iPrevId = pRoot->iDocid;
}while( pCsr->isEof==0
&& pRoot->eType==FTSQUERY_NEAR
- && fts3EvalTestDeferredAndNear(pCsr, &rc)
+ && sqlite3Fts3EvalTestDeferred(pCsr, &rc)
);
if( rc==SQLITE_OK && pCsr->isEof==0 ){
@@ -5602,7 +5648,6 @@ static int fts3EvalGatherStats(
fts3EvalNextRow(pCsr, pRoot, &rc);
assert( pRoot->bEof==0 );
}while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
- fts3EvalTestDeferredAndNear(pCsr, &rc);
}
}
return rc;
@@ -5712,10 +5757,10 @@ int sqlite3Fts3EvalPhrasePoslist(
int rc = SQLITE_OK;
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int bOr = 0;
- u8 bEof = 0;
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
+ int bMatch;
/* Check if this phrase descends from an OR expression node. If not,
** return NULL. Otherwise, the entry that corresponds to docid
@@ -5749,30 +5794,47 @@ int sqlite3Fts3EvalPhrasePoslist(
}
if( rc!=SQLITE_OK ) return rc;
- pIter = pPhrase->pOrPoslist;
- iDocid = pPhrase->iOrDocid;
- if( pCsr->bDesc==bDescDoclist ){
- bEof = (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
- while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
- sqlite3Fts3DoclistNext(
- bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
- &pIter, &iDocid, &bEof
- );
- }
- }else{
- bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
- while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
- int dummy;
- sqlite3Fts3DoclistPrev(
- bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
- &pIter, &iDocid, &dummy, &bEof
- );
+ bMatch = 1;
+ for(p=pNear; p; p=p->pLeft){
+ u8 bEof = 0;
+ Fts3Expr *pTest = p;
+ Fts3Phrase *pPh;
+ assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
+ if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
+ assert( pTest->eType==FTSQUERY_PHRASE );
+ pPh = pTest->pPhrase;
+
+ pIter = pPh->pOrPoslist;
+ iDocid = pPh->iOrDocid;
+ if( pCsr->bDesc==bDescDoclist ){
+ bEof = !pPh->doclist.nAll ||
+ (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
+ sqlite3Fts3DoclistNext(
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
+ &pIter, &iDocid, &bEof
+ );
+ }
+ }else{
+ bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
+ int dummy;
+ sqlite3Fts3DoclistPrev(
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
+ &pIter, &iDocid, &dummy, &bEof
+ );
+ }
}
+ pPh->pOrPoslist = pIter;
+ pPh->iOrDocid = iDocid;
+ if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
}
- pPhrase->pOrPoslist = pIter;
- pPhrase->iOrDocid = iDocid;
- if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
+ if( bMatch ){
+ pIter = pPhrase->pOrPoslist;
+ }else{
+ pIter = 0;
+ }
}
if( pIter==0 ) return SQLITE_OK;
diff --git a/lib/libsqlite3/ext/fts3/fts3Int.h b/lib/libsqlite3/ext/fts3/fts3Int.h
index 1b8b7bd97e9..981c37deee5 100644
--- a/lib/libsqlite3/ext/fts3/fts3Int.h
+++ b/lib/libsqlite3/ext/fts3/fts3Int.h
@@ -134,6 +134,11 @@ SQLITE_EXTENSION_INIT3
#ifdef SQLITE_COVERAGE_TEST
# define ALWAYS(x) (1)
# define NEVER(X) (0)
+#elif defined(SQLITE_DEBUG)
+# define ALWAYS(x) sqlite3Fts3Always((x)!=0)
+# define NEVER(x) sqlite3Fts3Never((x)!=0)
+int sqlite3Fts3Always(int b);
+int sqlite3Fts3Never(int b);
#else
# define ALWAYS(x) (x)
# define NEVER(x) (x)
@@ -192,6 +197,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
+typedef struct MatchinfoBuffer MatchinfoBuffer;
+
/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
@@ -301,9 +308,7 @@ struct Fts3Cursor {
i64 iMinDocid; /* Minimum docid to return */
i64 iMaxDocid; /* Maximum docid to return */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
- u32 *aMatchinfo; /* Information about most recent match */
- int nMatchinfo; /* Number of elements in aMatchinfo[] */
- char *zMatchinfo; /* Matchinfo specification */
+ MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
};
#define FTS3_EVAL_FILTER 0
@@ -423,7 +428,9 @@ struct Fts3Expr {
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
- u32 *aMI;
+ /* The following are used by the fts3_snippet.c module. */
+ int iPhrase; /* Index of this phrase in matchinfo() results */
+ u32 *aMI; /* See above */
};
/*
@@ -534,6 +541,7 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
)
/* fts3.c */
+void sqlite3Fts3ErrMsg(char**,const char*,...);
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
@@ -543,6 +551,7 @@ void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
+int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@@ -558,6 +567,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
+void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
diff --git a/lib/libsqlite3/ext/fts3/fts3_aux.c b/lib/libsqlite3/ext/fts3/fts3_aux.c
index c68b1a9d9be..f85a48ae020 100644
--- a/lib/libsqlite3/ext/fts3/fts3_aux.c
+++ b/lib/libsqlite3/ext/fts3/fts3_aux.c
@@ -116,7 +116,7 @@ static int fts3auxConnectMethod(
return SQLITE_OK;
bad_args:
- *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor");
+ sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor");
return SQLITE_ERROR;
}
diff --git a/lib/libsqlite3/ext/fts3/fts3_expr.c b/lib/libsqlite3/ext/fts3/fts3_expr.c
index 2ba786ce809..d7cabd99195 100644
--- a/lib/libsqlite3/ext/fts3/fts3_expr.c
+++ b/lib/libsqlite3/ext/fts3/fts3_expr.c
@@ -1022,13 +1022,13 @@ int sqlite3Fts3ExprParse(
sqlite3Fts3ExprFree(*ppExpr);
*ppExpr = 0;
if( rc==SQLITE_TOOBIG ){
- *pzErr = sqlite3_mprintf(
+ sqlite3Fts3ErrMsg(pzErr,
"FTS expression tree is too large (maximum depth %d)",
SQLITE_FTS3_MAX_EXPR_DEPTH
);
rc = SQLITE_ERROR;
}else if( rc==SQLITE_ERROR ){
- *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z);
+ sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z);
}
}
diff --git a/lib/libsqlite3/ext/fts3/fts3_icu.c b/lib/libsqlite3/ext/fts3/fts3_icu.c
index 52df8c7d812..6f90e1ebad0 100644
--- a/lib/libsqlite3/ext/fts3/fts3_icu.c
+++ b/lib/libsqlite3/ext/fts3/fts3_icu.c
@@ -240,12 +240,13 @@ static int icuNext(
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
- 0, /* iVersion */
- icuCreate, /* xCreate */
- icuDestroy, /* xCreate */
- icuOpen, /* xOpen */
- icuClose, /* xClose */
- icuNext, /* xNext */
+ 0, /* iVersion */
+ icuCreate, /* xCreate */
+ icuDestroy, /* xCreate */
+ icuOpen, /* xOpen */
+ icuClose, /* xClose */
+ icuNext, /* xNext */
+ 0, /* xLanguageid */
};
/*
diff --git a/lib/libsqlite3/ext/fts3/fts3_snippet.c b/lib/libsqlite3/ext/fts3/fts3_snippet.c
index 7933e29a7b7..a0771c0b305 100644
--- a/lib/libsqlite3/ext/fts3/fts3_snippet.c
+++ b/lib/libsqlite3/ext/fts3/fts3_snippet.c
@@ -27,6 +27,8 @@
#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
#define FTS3_MATCHINFO_LCS 's' /* nCol values */
#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
+#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */
+#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */
/*
** The default value for the second argument to matchinfo().
@@ -88,9 +90,22 @@ struct MatchInfo {
int nCol; /* Number of columns in table */
int nPhrase; /* Number of matchable phrases in query */
sqlite3_int64 nDoc; /* Number of docs in database */
+ char flag;
u32 *aMatchinfo; /* Pre-allocated buffer */
};
+/*
+** An instance of this structure is used to manage a pair of buffers, each
+** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
+** for details.
+*/
+struct MatchinfoBuffer {
+ u8 aRef[3];
+ int nElem;
+ int bGlobal; /* Set if global data is loaded */
+ char *zMatchinfo;
+ u32 aMatchinfo[1];
+};
/*
@@ -106,6 +121,97 @@ struct StrBuffer {
};
+/*************************************************************************
+** Start of MatchinfoBuffer code.
+*/
+
+/*
+** Allocate a two-slot MatchinfoBuffer object.
+*/
+static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
+ MatchinfoBuffer *pRet;
+ int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
+ int nStr = (int)strlen(zMatchinfo);
+
+ pRet = sqlite3_malloc(nByte + nStr+1);
+ if( pRet ){
+ memset(pRet, 0, nByte);
+ pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
+ pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
+ pRet->nElem = nElem;
+ pRet->zMatchinfo = ((char*)pRet) + nByte;
+ memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
+ pRet->aRef[0] = 1;
+ }
+
+ return pRet;
+}
+
+static void fts3MIBufferFree(void *p){
+ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
+
+ assert( (u32*)p==&pBuf->aMatchinfo[1]
+ || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2]
+ );
+ if( (u32*)p==&pBuf->aMatchinfo[1] ){
+ pBuf->aRef[1] = 0;
+ }else{
+ pBuf->aRef[2] = 0;
+ }
+
+ if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
+ sqlite3_free(pBuf);
+ }
+}
+
+static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
+ void (*xRet)(void*) = 0;
+ u32 *aOut = 0;
+
+ if( p->aRef[1]==0 ){
+ p->aRef[1] = 1;
+ aOut = &p->aMatchinfo[1];
+ xRet = fts3MIBufferFree;
+ }
+ else if( p->aRef[2]==0 ){
+ p->aRef[2] = 1;
+ aOut = &p->aMatchinfo[p->nElem+2];
+ xRet = fts3MIBufferFree;
+ }else{
+ aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
+ if( aOut ){
+ xRet = sqlite3_free;
+ if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
+ }
+ }
+
+ *paOut = aOut;
+ return xRet;
+}
+
+static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
+ p->bGlobal = 1;
+ memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
+}
+
+/*
+** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
+*/
+void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
+ if( p ){
+ assert( p->aRef[0]==1 );
+ p->aRef[0] = 0;
+ if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
+ sqlite3_free(p);
+ }
+ }
+}
+
+/*
+** End of MatchinfoBuffer code.
+*************************************************************************/
+
+
/*
** This function is used to help iterate through a position-list. A position
** list is a list of unique integers, sorted from smallest to largest. Each
@@ -142,7 +248,7 @@ static int fts3ExprIterate2(
void *pCtx /* Second argument to pass to callback */
){
int rc; /* Return code */
- int eType = pExpr->eType; /* Type of expression node pExpr */
+ int eType = pExpr->eType; /* Type of expression node pExpr */
if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
@@ -176,6 +282,7 @@ static int fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
+
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
@@ -220,8 +327,7 @@ static int fts3ExprLoadDoclists(
static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
(*(int *)ctx)++;
- UNUSED_PARAMETER(pExpr);
- UNUSED_PARAMETER(iPhrase);
+ pExpr->iPhrase = iPhrase;
return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
@@ -442,7 +548,7 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
- rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
+ rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
if( rc==SQLITE_OK ){
/* Set the *pmSeen output variable. */
@@ -744,6 +850,60 @@ static int fts3ColumnlistCount(char **ppCollist){
}
/*
+** This function gathers 'y' or 'b' data for a single phrase.
+*/
+static void fts3ExprLHits(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ MatchInfo *p /* Matchinfo context */
+){
+ Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
+ int iStart;
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ char *pIter = pPhrase->doclist.pList;
+ int iCol = 0;
+
+ assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
+ if( p->flag==FTS3_MATCHINFO_LHITS ){
+ iStart = pExpr->iPhrase * p->nCol;
+ }else{
+ iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
+ }
+
+ while( 1 ){
+ int nHit = fts3ColumnlistCount(&pIter);
+ if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
+ if( p->flag==FTS3_MATCHINFO_LHITS ){
+ p->aMatchinfo[iStart + iCol] = (u32)nHit;
+ }else if( nHit ){
+ p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
+ }
+ }
+ assert( *pIter==0x00 || *pIter==0x01 );
+ if( *pIter!=0x01 ) break;
+ pIter++;
+ pIter += fts3GetVarint32(pIter, &iCol);
+ }
+}
+
+/*
+** Gather the results for matchinfo directives 'y' and 'b'.
+*/
+static void fts3ExprLHitGather(
+ Fts3Expr *pExpr,
+ MatchInfo *p
+){
+ assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
+ if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
+ if( pExpr->pLeft ){
+ fts3ExprLHitGather(pExpr->pLeft, p);
+ fts3ExprLHitGather(pExpr->pRight, p);
+ }else{
+ fts3ExprLHits(pExpr, p);
+ }
+ }
+}
+
+/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
**
@@ -821,10 +981,12 @@ static int fts3MatchinfoCheck(
|| (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
|| (cArg==FTS3_MATCHINFO_LCS)
|| (cArg==FTS3_MATCHINFO_HITS)
+ || (cArg==FTS3_MATCHINFO_LHITS)
+ || (cArg==FTS3_MATCHINFO_LHITS_BM)
){
return SQLITE_OK;
}
- *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg);
return SQLITE_ERROR;
}
@@ -844,6 +1006,14 @@ static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
nVal = pInfo->nCol;
break;
+ case FTS3_MATCHINFO_LHITS:
+ nVal = pInfo->nCol * pInfo->nPhrase;
+ break;
+
+ case FTS3_MATCHINFO_LHITS_BM:
+ nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32);
+ break;
+
default:
assert( cArg==FTS3_MATCHINFO_HITS );
nVal = pInfo->nCol * pInfo->nPhrase * 3;
@@ -1038,7 +1208,7 @@ static int fts3MatchinfoValues(
sqlite3_stmt *pSelect = 0;
for(i=0; rc==SQLITE_OK && zArg[i]; i++){
-
+ pInfo->flag = zArg[i];
switch( zArg[i] ){
case FTS3_MATCHINFO_NPHRASE:
if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
@@ -1098,6 +1268,14 @@ static int fts3MatchinfoValues(
}
break;
+ case FTS3_MATCHINFO_LHITS_BM:
+ case FTS3_MATCHINFO_LHITS: {
+ int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
+ memset(pInfo->aMatchinfo, 0, nZero);
+ fts3ExprLHitGather(pCsr->pExpr, pInfo);
+ break;
+ }
+
default: {
Fts3Expr *pExpr;
assert( zArg[i]==FTS3_MATCHINFO_HITS );
@@ -1110,6 +1288,7 @@ static int fts3MatchinfoValues(
if( rc!=SQLITE_OK ) break;
}
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+ sqlite3Fts3EvalTestDeferred(pCsr, &rc);
if( rc!=SQLITE_OK ) break;
}
(void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
@@ -1129,7 +1308,8 @@ static int fts3MatchinfoValues(
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
-static int fts3GetMatchinfo(
+static void fts3GetMatchinfo(
+ sqlite3_context *pCtx, /* Return results here */
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
@@ -1138,6 +1318,9 @@ static int fts3GetMatchinfo(
int rc = SQLITE_OK;
int bGlobal = 0; /* Collect 'global' stats as well as local */
+ u32 *aOut = 0;
+ void (*xDestroyOut)(void*) = 0;
+
memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
@@ -1145,21 +1328,18 @@ static int fts3GetMatchinfo(
/* If there is cached matchinfo() data, but the format string for the
** cache does not match the format string for this request, discard
** the cached data. */
- if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
- assert( pCsr->aMatchinfo );
- sqlite3_free(pCsr->aMatchinfo);
- pCsr->zMatchinfo = 0;
- pCsr->aMatchinfo = 0;
+ if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+ pCsr->pMIBuffer = 0;
}
- /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+ /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
- if( pCsr->aMatchinfo==0 ){
+ if( pCsr->pMIBuffer==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
- int nArg; /* Bytes in zArg */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
@@ -1168,30 +1348,46 @@ static int fts3GetMatchinfo(
/* Determine the number of integers in the buffer returned by this call. */
for(i=0; zArg[i]; i++){
+ char *zErr = 0;
+ if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
+ sqlite3_result_error(pCtx, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
}
/* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
- nArg = (int)strlen(zArg);
- pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
- if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
-
- pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
- pCsr->nMatchinfo = nMatchinfo;
- memcpy(pCsr->zMatchinfo, zArg, nArg+1);
- memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+ pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
+ if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
+
pCsr->isMatchinfoNeeded = 1;
bGlobal = 1;
}
- sInfo.aMatchinfo = pCsr->aMatchinfo;
- sInfo.nPhrase = pCsr->nPhrase;
- if( pCsr->isMatchinfoNeeded ){
+ if( rc==SQLITE_OK ){
+ xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
+ if( xDestroyOut==0 ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ sInfo.aMatchinfo = aOut;
+ sInfo.nPhrase = pCsr->nPhrase;
rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
- pCsr->isMatchinfoNeeded = 0;
+ if( bGlobal ){
+ fts3MIBufferSetGlobal(pCsr->pMIBuffer);
+ }
}
- return rc;
+ if( rc!=SQLITE_OK ){
+ sqlite3_result_error_code(pCtx, rc);
+ if( xDestroyOut ) xDestroyOut(aOut);
+ }else{
+ int n = pCsr->pMIBuffer->nElem * sizeof(u32);
+ sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
+ }
}
/*
@@ -1397,7 +1593,7 @@ void sqlite3Fts3Offsets(
*/
sCtx.iCol = iCol;
sCtx.iTerm = 0;
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
+ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
/* Retreive the text stored in column iCol. If an SQL NULL is stored
** in column iCol, jump immediately to the next iteration of the loop.
@@ -1489,19 +1685,9 @@ void sqlite3Fts3Matchinfo(
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
- int rc;
- int i;
const char *zFormat;
if( zArg ){
- for(i=0; zArg[i]; i++){
- char *zErr = 0;
- if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
- sqlite3_result_error(pContext, zErr, -1);
- sqlite3_free(zErr);
- return;
- }
- }
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
@@ -1510,17 +1696,10 @@ void sqlite3Fts3Matchinfo(
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
- }
-
- /* Retrieve matchinfo() data. */
- rc = fts3GetMatchinfo(pCsr, zFormat);
- sqlite3Fts3SegmentsClose(pTab);
-
- if( rc!=SQLITE_OK ){
- sqlite3_result_error_code(pContext, rc);
}else{
- int n = pCsr->nMatchinfo * sizeof(u32);
- sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
+ /* Retrieve matchinfo() data. */
+ fts3GetMatchinfo(pContext, pCsr, zFormat);
+ sqlite3Fts3SegmentsClose(pTab);
}
}
diff --git a/lib/libsqlite3/ext/fts3/fts3_term.c b/lib/libsqlite3/ext/fts3/fts3_term.c
index c49d5cb65d7..7edd0728925 100644
--- a/lib/libsqlite3/ext/fts3/fts3_term.c
+++ b/lib/libsqlite3/ext/fts3/fts3_term.c
@@ -81,7 +81,7 @@ static int fts3termConnectMethod(
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
- *pzErr = sqlite3_mprintf(
+ sqlite3Fts3ErrMsg(pzErr,
"wrong number of arguments to fts4term constructor"
);
return SQLITE_ERROR;
diff --git a/lib/libsqlite3/ext/fts3/fts3_tokenize_vtab.c b/lib/libsqlite3/ext/fts3/fts3_tokenize_vtab.c
index fb99f8b8064..dfeddfeb963 100644
--- a/lib/libsqlite3/ext/fts3/fts3_tokenize_vtab.c
+++ b/lib/libsqlite3/ext/fts3/fts3_tokenize_vtab.c
@@ -85,7 +85,7 @@ static int fts3tokQueryTokenizer(
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName);
return SQLITE_ERROR;
}
diff --git a/lib/libsqlite3/ext/fts3/fts3_tokenizer.c b/lib/libsqlite3/ext/fts3/fts3_tokenizer.c
index 2b985f5f33d..64cfe07aac3 100644
--- a/lib/libsqlite3/ext/fts3/fts3_tokenizer.c
+++ b/lib/libsqlite3/ext/fts3/fts3_tokenizer.c
@@ -172,7 +172,7 @@ int sqlite3Fts3InitTokenizer(
m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
}else{
char const **aArg = 0;
@@ -195,7 +195,7 @@ int sqlite3Fts3InitTokenizer(
rc = m->xCreate(iArg, aArg, ppTok);
assert( rc!=SQLITE_OK || *ppTok );
if( rc!=SQLITE_OK ){
- *pzErr = sqlite3_mprintf("unknown tokenizer");
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer");
}else{
(*ppTok)->pModule = m;
}
@@ -279,9 +279,9 @@ static void testFunc(
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
- char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
- sqlite3_result_error(context, zErr, -1);
- sqlite3_free(zErr);
+ char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr2, -1);
+ sqlite3_free(zErr2);
return;
}
diff --git a/lib/libsqlite3/ext/fts3/fts3_write.c b/lib/libsqlite3/ext/fts3/fts3_write.c
index a16070766ff..4cd2aebf6a8 100644
--- a/lib/libsqlite3/ext/fts3/fts3_write.c
+++ b/lib/libsqlite3/ext/fts3/fts3_write.c
@@ -326,7 +326,7 @@ static int fts3SqlStmt(
/* 25 */ "",
/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
-/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
+/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
@@ -3444,7 +3444,8 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
int rc2;
- sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+ sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
+ sqlite3_bind_int(pAllLangid, 2, p->nIndex);
while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
int i;
int iLangid = sqlite3_column_int(pAllLangid, 0);
@@ -4776,7 +4777,7 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
pHint->n = i;
i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
i += fts3GetVarint32(&pHint->a[i], pnInput);
- if( i!=nHint ) return SQLITE_CORRUPT_VTAB;
+ if( i!=nHint ) return FTS_CORRUPT_VTAB;
return SQLITE_OK;
}
@@ -5144,7 +5145,8 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
int rc2;
- sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+ sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
+ sqlite3_bind_int(pAllLangid, 2, p->nIndex);
while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
int iLangid = sqlite3_column_int(pAllLangid, 0);
int i;
@@ -5157,7 +5159,6 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
}
/* This block calculates the checksum according to the %_content table */
- rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
if( rc==SQLITE_OK ){
sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
sqlite3_stmt *pStmt = 0;
@@ -5254,7 +5255,7 @@ static int fts3DoIntegrityCheck(
int rc;
int bOk = 0;
rc = fts3IntegrityCheck(p, &bOk);
- if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
+ if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
return rc;
}
diff --git a/lib/libsqlite3/ext/fts3/unicode/mkunicode.tcl b/lib/libsqlite3/ext/fts3/unicode/mkunicode.tcl
index c3083ee3686..a2e9b1da293 100644
--- a/lib/libsqlite3/ext/fts3/unicode/mkunicode.tcl
+++ b/lib/libsqlite3/ext/fts3/unicode/mkunicode.tcl
@@ -1,77 +1,5 @@
-#
-# Parameter $zName must be a path to the file UnicodeData.txt. This command
-# reads the file and returns a list of mappings required to remove all
-# diacritical marks from a unicode string. Each mapping is itself a list
-# consisting of two elements - the unicode codepoint and the single ASCII
-# character that it should be replaced with, or an empty string if the
-# codepoint should simply be removed from the input. Examples:
-#
-# { 224 a } (replace codepoint 224 to "a")
-# { 769 "" } (remove codepoint 769 from input)
-#
-# Mappings are only returned for non-upper case codepoints. It is assumed
-# that the input has already been folded to lower case.
-#
-proc rd_load_unicodedata_text {zName} {
- global tl_lookup_table
-
- set fd [open $zName]
- set lField {
- code
- character_name
- general_category
- canonical_combining_classes
- bidirectional_category
- character_decomposition_mapping
- decimal_digit_value
- digit_value
- numeric_value
- mirrored
- unicode_1_name
- iso10646_comment_field
- uppercase_mapping
- lowercase_mapping
- titlecase_mapping
- }
- set lRet [list]
-
- while { ![eof $fd] } {
- set line [gets $fd]
- if {$line == ""} continue
-
- set fields [split $line ";"]
- if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
- foreach $lField $fields {}
- if { [llength $character_decomposition_mapping]!=2
- || [string is xdigit [lindex $character_decomposition_mapping 0]]==0
- } {
- continue
- }
-
- set iCode [expr "0x$code"]
- set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
- set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
-
- if {[info exists tl_lookup_table($iCode)]} continue
-
- if { ($iAscii >= 97 && $iAscii <= 122)
- || ($iAscii >= 65 && $iAscii <= 90)
- } {
- lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
- set dia($iDia) 1
- }
- }
-
- foreach d [array names dia] {
- lappend lRet [list $d ""]
- }
- set lRet [lsort -integer -index 0 $lRet]
-
- close $fd
- set lRet
-}
-
+source [file join [file dirname [info script]] parseunicode.tcl]
proc print_rd {map} {
global tl_lookup_table
@@ -117,7 +45,7 @@ proc print_rd {map} {
puts "** E\"). The resuls of passing a codepoint that corresponds to an"
puts "** uppercase letter are undefined."
puts "*/"
- puts "static int remove_diacritic(int c)\{"
+ puts "static int ${::remove_diacritic}(int c)\{"
puts " unsigned short aDia\[\] = \{"
puts -nonewline " 0, "
set i 1
@@ -204,53 +132,6 @@ proc print_isdiacritic {zFunc map} {
#-------------------------------------------------------------------------
-# Parameter $zName must be a path to the file UnicodeData.txt. This command
-# reads the file and returns a list of codepoints (integers). The list
-# contains all codepoints in the UnicodeData.txt assigned to any "General
-# Category" that is not a "Letter" or "Number".
-#
-proc an_load_unicodedata_text {zName} {
- set fd [open $zName]
- set lField {
- code
- character_name
- general_category
- canonical_combining_classes
- bidirectional_category
- character_decomposition_mapping
- decimal_digit_value
- digit_value
- numeric_value
- mirrored
- unicode_1_name
- iso10646_comment_field
- uppercase_mapping
- lowercase_mapping
- titlecase_mapping
- }
- set lRet [list]
-
- while { ![eof $fd] } {
- set line [gets $fd]
- if {$line == ""} continue
-
- set fields [split $line ";"]
- if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
- foreach $lField $fields {}
-
- set iCode [expr "0x$code"]
- set bAlnum [expr {
- [lsearch {L N} [string range $general_category 0 0]] >= 0
- || $general_category=="Co"
- }]
-
- if { !$bAlnum } { lappend lRet $iCode }
- }
-
- close $fd
- set lRet
-}
-
proc an_load_separator_ranges {} {
global unicodedata.txt
set lSep [an_load_unicodedata_text ${unicodedata.txt}]
@@ -440,29 +321,6 @@ proc print_test_isalnum {zFunc lRange} {
#-------------------------------------------------------------------------
-proc tl_load_casefolding_txt {zName} {
- global tl_lookup_table
-
- set fd [open $zName]
- while { ![eof $fd] } {
- set line [gets $fd]
- if {[string range $line 0 0] == "#"} continue
- if {$line == ""} continue
-
- foreach x {a b c d} {unset -nocomplain $x}
- foreach {a b c d} [split $line ";"] {}
-
- set a2 [list]
- set c2 [list]
- foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
- foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
- set b [string trim $b]
- set d [string trim $d]
-
- if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
- }
-}
-
proc tl_create_records {} {
global tl_lookup_table
@@ -626,19 +484,20 @@ proc print_fold {zFunc} {
tl_print_table_footer toggle
tl_print_ioff_table $liOff
- puts {
+ puts [subst -nocommands {
int ret = c;
- assert( c>=0 );
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
+ const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
+ assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
@@ -649,19 +508,17 @@ proc print_fold {zFunc} {
iHi = iTest-1;
}
}
- assert( iRes<0 || c>=aEntry[iRes].iCode );
- if( iRes>=0 ){
- const struct TableEntry *p = &aEntry[iRes];
- if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
- ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
- assert( ret>0 );
- }
+ assert( iRes>=0 && c>=aEntry[iRes].iCode );
+ p = &aEntry[iRes];
+ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
+ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
+ assert( ret>0 );
}
- if( bRemoveDiacritic ) ret = remove_diacritic(ret);
- }
+ if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
}
+ }]
foreach entry $lHigh {
tl_print_if_entry $entry
@@ -732,8 +589,12 @@ proc print_fileheader {} {
*/
}]
puts ""
- puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
- puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
+ if {$::generate_fts5_code} {
+ # no-op
+ } else {
+ puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
+ puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
+ }
puts ""
puts "#include <assert.h>"
puts ""
@@ -760,22 +621,40 @@ proc print_test_main {} {
# our liking.
#
proc usage {} {
- puts -nonewline stderr "Usage: $::argv0 ?-test? "
+ puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? "
puts stderr "<CaseFolding.txt file> <UnicodeData.txt file>"
exit 1
}
-if {[llength $argv]!=2 && [llength $argv]!=3} usage
-if {[llength $argv]==3 && [lindex $argv 0]!="-test"} usage
+if {[llength $argv]<2} usage
set unicodedata.txt [lindex $argv end]
set casefolding.txt [lindex $argv end-1]
-set generate_test_code [expr {[llength $argv]==3}]
+
+set remove_diacritic remove_diacritic
+set generate_test_code 0
+set generate_fts5_code 0
+set function_prefix "sqlite3Fts"
+for {set i 0} {$i < [llength $argv]-2} {incr i} {
+ switch -- [lindex $argv $i] {
+ -test {
+ set generate_test_code 1
+ }
+ -fts5 {
+ set function_prefix sqlite3Fts5
+ set generate_fts5_code 1
+ set remove_diacritic fts5_remove_diacritic
+ }
+ default {
+ usage
+ }
+ }
+}
print_fileheader
# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
-print_isalnum sqlite3FtsUnicodeIsalnum $lRange
+print_isalnum ${function_prefix}UnicodeIsalnum $lRange
# Leave a gap between the two generated C functions.
#
@@ -790,22 +669,26 @@ set mappings [rd_load_unicodedata_text ${unicodedata.txt}]
print_rd $mappings
puts ""
puts ""
-print_isdiacritic sqlite3FtsUnicodeIsdiacritic $mappings
+print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
puts ""
puts ""
# Print the fold() function to stdout.
#
-print_fold sqlite3FtsUnicodeFold
+print_fold ${function_prefix}UnicodeFold
# Print the test routines and main() function to stdout, if -test
# was specified.
#
if {$::generate_test_code} {
- print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange
- print_fold_test sqlite3FtsUnicodeFold $mappings
+ print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+ print_fold_test ${function_prefix}UnicodeFold $mappings
print_test_main
}
-puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
-puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
+if {$generate_fts5_code} {
+ # no-op
+} else {
+ puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
+ puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
+}
diff --git a/lib/libsqlite3/ext/fts3/unicode/parseunicode.tcl b/lib/libsqlite3/ext/fts3/unicode/parseunicode.tcl
new file mode 100644
index 00000000000..0cb2c83a18f
--- /dev/null
+++ b/lib/libsqlite3/ext/fts3/unicode/parseunicode.tcl
@@ -0,0 +1,146 @@
+
+#--------------------------------------------------------------------------
+# Parameter $zName must be a path to the file UnicodeData.txt. This command
+# reads the file and returns a list of mappings required to remove all
+# diacritical marks from a unicode string. Each mapping is itself a list
+# consisting of two elements - the unicode codepoint and the single ASCII
+# character that it should be replaced with, or an empty string if the
+# codepoint should simply be removed from the input. Examples:
+#
+# { 224 a } (replace codepoint 224 to "a")
+# { 769 "" } (remove codepoint 769 from input)
+#
+# Mappings are only returned for non-upper case codepoints. It is assumed
+# that the input has already been folded to lower case.
+#
+proc rd_load_unicodedata_text {zName} {
+ global tl_lookup_table
+
+ set fd [open $zName]
+ set lField {
+ code
+ character_name
+ general_category
+ canonical_combining_classes
+ bidirectional_category
+ character_decomposition_mapping
+ decimal_digit_value
+ digit_value
+ numeric_value
+ mirrored
+ unicode_1_name
+ iso10646_comment_field
+ uppercase_mapping
+ lowercase_mapping
+ titlecase_mapping
+ }
+ set lRet [list]
+
+ while { ![eof $fd] } {
+ set line [gets $fd]
+ if {$line == ""} continue
+
+ set fields [split $line ";"]
+ if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+ foreach $lField $fields {}
+ if { [llength $character_decomposition_mapping]!=2
+ || [string is xdigit [lindex $character_decomposition_mapping 0]]==0
+ } {
+ continue
+ }
+
+ set iCode [expr "0x$code"]
+ set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
+ set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
+
+ if {[info exists tl_lookup_table($iCode)]} continue
+
+ if { ($iAscii >= 97 && $iAscii <= 122)
+ || ($iAscii >= 65 && $iAscii <= 90)
+ } {
+ lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+ set dia($iDia) 1
+ }
+ }
+
+ foreach d [array names dia] {
+ lappend lRet [list $d ""]
+ }
+ set lRet [lsort -integer -index 0 $lRet]
+
+ close $fd
+ set lRet
+}
+
+#-------------------------------------------------------------------------
+# Parameter $zName must be a path to the file UnicodeData.txt. This command
+# reads the file and returns a list of codepoints (integers). The list
+# contains all codepoints in the UnicodeData.txt assigned to any "General
+# Category" that is not a "Letter" or "Number".
+#
+proc an_load_unicodedata_text {zName} {
+ set fd [open $zName]
+ set lField {
+ code
+ character_name
+ general_category
+ canonical_combining_classes
+ bidirectional_category
+ character_decomposition_mapping
+ decimal_digit_value
+ digit_value
+ numeric_value
+ mirrored
+ unicode_1_name
+ iso10646_comment_field
+ uppercase_mapping
+ lowercase_mapping
+ titlecase_mapping
+ }
+ set lRet [list]
+
+ while { ![eof $fd] } {
+ set line [gets $fd]
+ if {$line == ""} continue
+
+ set fields [split $line ";"]
+ if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+ foreach $lField $fields {}
+
+ set iCode [expr "0x$code"]
+ set bAlnum [expr {
+ [lsearch {L N} [string range $general_category 0 0]] >= 0
+ || $general_category=="Co"
+ }]
+
+ if { !$bAlnum } { lappend lRet $iCode }
+ }
+
+ close $fd
+ set lRet
+}
+
+proc tl_load_casefolding_txt {zName} {
+ global tl_lookup_table
+
+ set fd [open $zName]
+ while { ![eof $fd] } {
+ set line [gets $fd]
+ if {[string range $line 0 0] == "#"} continue
+ if {$line == ""} continue
+
+ foreach x {a b c d} {unset -nocomplain $x}
+ foreach {a b c d} [split $line ";"] {}
+
+ set a2 [list]
+ set c2 [list]
+ foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
+ foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
+ set b [string trim $b]
+ set d [string trim $d]
+
+ if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
+ }
+}
+
+
diff --git a/lib/libsqlite3/ext/icu/icu.c b/lib/libsqlite3/ext/icu/icu.c
index 7cc79b21d00..a9f23691045 100644
--- a/lib/libsqlite3/ext/icu/icu.c
+++ b/lib/libsqlite3/ext/icu/icu.c
@@ -9,7 +9,7 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
-** $Id: icu.c,v 1.1.1.2 2013/09/21 17:29:28 jturner Exp $
+** $Id: icu.c,v 1.2 2015/09/12 02:08:35 jturner Exp $
**
** This file implements an integration between the ICU library
** ("International Components for Unicode", an open-source library
@@ -83,7 +83,6 @@ static int icuLikeCompare(
/* Read (and consume) the next character from the input pattern. */
UChar32 uPattern;
U8_NEXT_UNSAFE(zPattern, iPattern, uPattern);
- assert(uPattern!=0);
/* There are now 4 possibilities:
**
@@ -422,6 +421,7 @@ static void icuLoadCollation(
int rc; /* Return code from sqlite3_create_collation_x() */
assert(nArg==2);
+ (void)nArg; /* Unused parameter */
zLocale = (const char *)sqlite3_value_text(apArg[0]);
zName = (const char *)sqlite3_value_text(apArg[1]);
diff --git a/lib/libsqlite3/ext/misc/fuzzer.c b/lib/libsqlite3/ext/misc/fuzzer.c
index dc03161aafa..3ed4b0a9772 100644
--- a/lib/libsqlite3/ext/misc/fuzzer.c
+++ b/lib/libsqlite3/ext/misc/fuzzer.c
@@ -876,7 +876,7 @@ static fuzzer_stem *fuzzerNewStem(
if( pNew==0 ) return 0;
memset(pNew, 0, sizeof(*pNew));
pNew->zBasis = (char*)&pNew[1];
- pNew->nBasis = (int)strlen(zWord);
+ pNew->nBasis = (fuzzer_len)strlen(zWord);
memcpy(pNew->zBasis, zWord, pNew->nBasis+1);
pRule = pCur->pVtab->pRule;
while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){
diff --git a/lib/libsqlite3/ext/misc/spellfix.c b/lib/libsqlite3/ext/misc/spellfix.c
index a6f780584c2..b9514427cf9 100644
--- a/lib/libsqlite3/ext/misc/spellfix.c
+++ b/lib/libsqlite3/ext/misc/spellfix.c
@@ -2656,6 +2656,31 @@ static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
}
/*
+** This function is called by the xUpdate() method. It returns a string
+** containing the conflict mode that xUpdate() should use for the current
+** operation. One of: "ROLLBACK", "IGNORE", "ABORT" or "REPLACE".
+*/
+static const char *spellfix1GetConflict(sqlite3 *db){
+ static const char *azConflict[] = {
+ /* Note: Instead of "FAIL" - "ABORT". */
+ "ROLLBACK", "IGNORE", "ABORT", "ABORT", "REPLACE"
+ };
+ int eConflict = sqlite3_vtab_on_conflict(db);
+
+ assert( eConflict==SQLITE_ROLLBACK || eConflict==SQLITE_IGNORE
+ || eConflict==SQLITE_FAIL || eConflict==SQLITE_ABORT
+ || eConflict==SQLITE_REPLACE
+ );
+ assert( SQLITE_ROLLBACK==1 );
+ assert( SQLITE_IGNORE==2 );
+ assert( SQLITE_FAIL==3 );
+ assert( SQLITE_ABORT==4 );
+ assert( SQLITE_REPLACE==5 );
+
+ return azConflict[eConflict-1];
+}
+
+/*
** The xUpdate() method.
*/
static int spellfix1Update(
@@ -2686,6 +2711,7 @@ static int spellfix1Update(
char *zK1, *zK2;
int i;
char c;
+ const char *zConflict = spellfix1GetConflict(db);
if( zWord==0 ){
/* Inserts of the form: INSERT INTO table(command) VALUES('xyzzy');
@@ -2746,10 +2772,10 @@ static int spellfix1Update(
}else{
newRowid = sqlite3_value_int64(argv[1]);
spellfix1DbExec(&rc, db,
- "INSERT INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
- "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
- p->zDbName, p->zTableName,
- newRowid, iRank, iLang, zWord, zK1, zK2
+ "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
+ "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
+ zConflict, p->zDbName, p->zTableName,
+ newRowid, iRank, iLang, zWord, zK1, zK2
);
}
*pRowid = sqlite3_last_insert_rowid(db);
@@ -2757,9 +2783,9 @@ static int spellfix1Update(
rowid = sqlite3_value_int64(argv[0]);
newRowid = *pRowid = sqlite3_value_int64(argv[1]);
spellfix1DbExec(&rc, db,
- "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
+ "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
" word=%Q, k1=%Q, k2=%Q WHERE id=%lld",
- p->zDbName, p->zTableName, newRowid, iRank, iLang,
+ zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang,
zWord, zK1, zK2, rowid
);
}
diff --git a/lib/libsqlite3/ext/rtree/rtree.c b/lib/libsqlite3/ext/rtree/rtree.c
index 201d3cfff2d..4e473a22c28 100644
--- a/lib/libsqlite3/ext/rtree/rtree.c
+++ b/lib/libsqlite3/ext/rtree/rtree.c
@@ -351,6 +351,7 @@ struct RtreeMatchArg {
u32 magic; /* Always RTREE_GEOMETRY_MAGIC */
RtreeGeomCallback cb; /* Info about the callback functions */
int nParam; /* Number of parameters to the SQL function */
+ sqlite3_value **apSqlParam; /* Original SQL parameter values */
RtreeDValue aParam[1]; /* Values for parameters to the SQL function */
};
@@ -1482,9 +1483,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
/* Check that the blob is roughly the right size. */
nBlob = sqlite3_value_bytes(pValue);
- if( nBlob<(int)sizeof(RtreeMatchArg)
- || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0
- ){
+ if( nBlob<(int)sizeof(RtreeMatchArg) ){
return SQLITE_ERROR;
}
@@ -1495,6 +1494,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
memcpy(pBlob, sqlite3_value_blob(pValue), nBlob);
nExpected = (int)(sizeof(RtreeMatchArg) +
+ pBlob->nParam*sizeof(sqlite3_value*) +
(pBlob->nParam-1)*sizeof(RtreeDValue));
if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){
sqlite3_free(pInfo);
@@ -1503,6 +1503,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
pInfo->pContext = pBlob->cb.pContext;
pInfo->nParam = pBlob->nParam;
pInfo->aParam = pBlob->aParam;
+ pInfo->apSqlParam = pBlob->apSqlParam;
if( pBlob->cb.xGeom ){
pCons->u.xGeom = pBlob->cb.xGeom;
@@ -1669,17 +1670,30 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
Rtree *pRtree = (Rtree*)tab;
int rc = SQLITE_OK;
int ii;
+ int bMatch = 0; /* True if there exists a MATCH constraint */
i64 nRow; /* Estimated rows returned by this scan */
int iIdx = 0;
char zIdxStr[RTREE_MAX_DIMENSIONS*8+1];
memset(zIdxStr, 0, sizeof(zIdxStr));
+ /* Check if there exists a MATCH constraint - even an unusable one. If there
+ ** is, do not consider the lookup-by-rowid plan as using such a plan would
+ ** require the VDBE to evaluate the MATCH constraint, which is not currently
+ ** possible. */
+ for(ii=0; ii<pIdxInfo->nConstraint; ii++){
+ if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){
+ bMatch = 1;
+ }
+ }
+
assert( pIdxInfo->idxStr==0 );
for(ii=0; ii<pIdxInfo->nConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){
struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii];
- if( p->usable && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ if( bMatch==0 && p->usable
+ && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+ ){
/* We have an equality constraint on the rowid. Use strategy 1. */
int jj;
for(jj=0; jj<ii; jj++){
@@ -2820,11 +2834,19 @@ static int rtreeUpdate(
if( nData>1 ){
int ii;
- /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */
- assert( nData==(pRtree->nDim*2 + 3) );
+ /* Populate the cell.aCoord[] array. The first coordinate is azData[3].
+ **
+ ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared
+ ** with "column" that are interpreted as table constraints.
+ ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5));
+ ** This problem was discovered after years of use, so we silently ignore
+ ** these kinds of misdeclared tables to avoid breaking any legacy.
+ */
+ assert( nData<=(pRtree->nDim*2 + 3) );
+
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
- for(ii=0; ii<(pRtree->nDim*2); ii+=2){
+ for(ii=0; ii<nData-4; ii+=2){
cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]);
cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]);
if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){
@@ -2835,7 +2857,7 @@ static int rtreeUpdate(
}else
#endif
{
- for(ii=0; ii<(pRtree->nDim*2); ii+=2){
+ for(ii=0; ii<nData-4; ii+=2){
cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]);
cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]);
if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){
@@ -3365,6 +3387,18 @@ static void rtreeFreeCallback(void *p){
}
/*
+** This routine frees the BLOB that is returned by geomCallback().
+*/
+static void rtreeMatchArgFree(void *pArg){
+ int i;
+ RtreeMatchArg *p = (RtreeMatchArg*)pArg;
+ for(i=0; i<p->nParam; i++){
+ sqlite3_value_free(p->apSqlParam[i]);
+ }
+ sqlite3_free(p);
+}
+
+/*
** Each call to sqlite3_rtree_geometry_callback() or
** sqlite3_rtree_query_callback() creates an ordinary SQLite
** scalar function that is implemented by this routine.
@@ -3382,8 +3416,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx);
RtreeMatchArg *pBlob;
int nBlob;
+ int memErr = 0;
- nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue);
+ nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue)
+ + nArg*sizeof(sqlite3_value*);
pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob);
if( !pBlob ){
sqlite3_result_error_nomem(ctx);
@@ -3391,15 +3427,23 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
int i;
pBlob->magic = RTREE_GEOMETRY_MAGIC;
pBlob->cb = pGeomCtx[0];
+ pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg];
pBlob->nParam = nArg;
for(i=0; i<nArg; i++){
+ pBlob->apSqlParam[i] = sqlite3_value_dup(aArg[i]);
+ if( pBlob->apSqlParam[i]==0 ) memErr = 1;
#ifdef SQLITE_RTREE_INT_ONLY
pBlob->aParam[i] = sqlite3_value_int64(aArg[i]);
#else
pBlob->aParam[i] = sqlite3_value_double(aArg[i]);
#endif
}
- sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free);
+ if( memErr ){
+ sqlite3_result_error_nomem(ctx);
+ rtreeMatchArgFree(pBlob);
+ }else{
+ sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree);
+ }
}
}
diff --git a/lib/libsqlite3/ext/rtree/rtree9.test b/lib/libsqlite3/ext/rtree/rtree9.test
index 6479516bed6..571341b2a93 100644
--- a/lib/libsqlite3/ext/rtree/rtree9.test
+++ b/lib/libsqlite3/ext/rtree/rtree9.test
@@ -87,7 +87,6 @@ do_catchsql_test rtree9-4.3 {
#
register_circle_geom db
-breakpoint
do_execsql_test rtree9-5.1 {
CREATE VIRTUAL TABLE rt2 USING rtree(id, xmin, xmax, ymin, ymax);
diff --git a/lib/libsqlite3/ext/rtree/rtreeC.test b/lib/libsqlite3/ext/rtree/rtreeC.test
index 94db05a4d1b..9a64df51d53 100644
--- a/lib/libsqlite3/ext/rtree/rtreeC.test
+++ b/lib/libsqlite3/ext/rtree/rtreeC.test
@@ -269,5 +269,88 @@ ifcapable rtree {
db close
}
+#--------------------------------------------------------------------
+# Test that queries featuring LEFT or CROSS JOINS are handled correctly.
+# Handled correctly in this case means:
+#
+# * Terms with prereqs that appear to the left of a LEFT JOIN against
+# the virtual table are always available to xBestIndex.
+#
+# * Terms with prereqs that appear to the right of a LEFT JOIN against
+# the virtual table are never available to xBestIndex.
+#
+# And the same behaviour for CROSS joins.
+#
+reset_db
+do_execsql_test 7.0 {
+ CREATE TABLE xdir(x1);
+ CREATE TABLE ydir(y1);
+ CREATE VIRTUAL TABLE rt USING rtree_i32(id, xmin, xmax, ymin, ymax);
+
+ INSERT INTO xdir VALUES(5);
+ INSERT INTO ydir VALUES(10);
+
+ INSERT INTO rt VALUES(1, 2, 7, 12, 14); -- Not a hit
+ INSERT INTO rt VALUES(2, 2, 7, 8, 12); -- A hit!
+ INSERT INTO rt VALUES(3, 7, 11, 8, 12); -- Not a hit!
+ INSERT INTO rt VALUES(4, 5, 5, 10, 10); -- A hit!
+
+}
+
+proc do_eqp_execsql_test {tn sql res} {
+ set query "EXPLAIN QUERY PLAN $sql ; $sql "
+ uplevel [list do_execsql_test $tn $query $res]
+}
+
+do_eqp_execsql_test 7.1 {
+ SELECT id FROM xdir, rt, ydir
+ ON (y1 BETWEEN ymin AND ymax)
+ WHERE (x1 BETWEEN xmin AND xmax);
+} {
+ 0 0 0 {SCAN TABLE xdir}
+ 0 1 2 {SCAN TABLE ydir}
+ 0 2 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B2D3B0D1}
+ 2 4
+}
+
+do_eqp_execsql_test 7.2 {
+ SELECT * FROM xdir, rt LEFT JOIN ydir
+ ON (y1 BETWEEN ymin AND ymax)
+ WHERE (x1 BETWEEN xmin AND xmax);
+} {
+ 0 0 0 {SCAN TABLE xdir}
+ 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
+ 0 2 2 {SCAN TABLE ydir}
+
+ 5 1 2 7 12 14 {}
+ 5 2 2 7 8 12 10
+ 5 4 5 5 10 10 10
+}
+
+do_eqp_execsql_test 7.3 {
+ SELECT id FROM xdir, rt CROSS JOIN ydir
+ ON (y1 BETWEEN ymin AND ymax)
+ WHERE (x1 BETWEEN xmin AND xmax);
+} {
+ 0 0 0 {SCAN TABLE xdir}
+ 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
+ 0 2 2 {SCAN TABLE ydir}
+ 2 4
+}
+
+do_eqp_execsql_test 7.4 {
+ SELECT id FROM rt, xdir CROSS JOIN ydir
+ ON (y1 BETWEEN ymin AND ymax)
+ WHERE (x1 BETWEEN xmin AND xmax);
+} {
+ 0 0 1 {SCAN TABLE xdir}
+ 0 1 0 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
+ 0 2 2 {SCAN TABLE ydir}
+ 2 4
+}
+
+finish_test
+
+
finish_test
diff --git a/lib/libsqlite3/ext/rtree/rtreeE.test b/lib/libsqlite3/ext/rtree/rtreeE.test
index c450623790e..3e5ba3a67fa 100644
--- a/lib/libsqlite3/ext/rtree/rtreeE.test
+++ b/lib/libsqlite3/ext/rtree/rtreeE.test
@@ -52,6 +52,9 @@ do_execsql_test rtreeE-1.1 {
do_execsql_test rtreeE-1.1 {
SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 0.0, 50.0, 3) ORDER BY id;
} {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24}
+do_execsql_test rtreeE-1.1x {
+ SELECT id FROM rt1 WHERE id MATCH Qcircle('x:0 y:0 r:50.0 e:3') ORDER BY id;
+} {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24}
do_execsql_test rtreeE-1.2 {
SELECT id FROM rt1 WHERE id MATCH Qcircle(100.0, 0.0, 50.0, 3) ORDER BY id;
} {100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124}
@@ -64,12 +67,12 @@ do_execsql_test rtreeE-1.3 {
# last.
#
do_execsql_test rtreeE-1.4 {
- SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,3) AND id%100==0
+ SELECT id FROM rt1 WHERE id MATCH Qcircle('r:1000 e:3') AND id%100==0
} {200 100 0}
# Exclude odd rowids on a depth-first search
do_execsql_test rtreeE-1.5 {
- SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,4) ORDER BY +id
+ SELECT id FROM rt1 WHERE id MATCH Qcircle('r:1000 e:4') ORDER BY +id
} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224}
# Exclude odd rowids on a breadth-first search.
@@ -77,6 +80,13 @@ do_execsql_test rtreeE-1.6 {
SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,5) ORDER BY +id
} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224}
+# Test that rtree prefers MATCH to lookup-by-rowid.
+#
+do_execsql_test rtreeE-1.7 {
+ SELECT id FROM rt1 WHERE id=18 AND id MATCH Qcircle(0,0,1000,5)
+} {18}
+
+
# Construct a large 2-D RTree with thousands of random entries.
#
do_test rtreeE-2.1 {
@@ -126,4 +136,5 @@ do_execsql_test rtreeE-2.4 {
SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,10000,0,10000) ORDER BY id
} $ans
+
finish_test
diff --git a/lib/libsqlite3/ext/rtree/sqlite3rtree.h b/lib/libsqlite3/ext/rtree/sqlite3rtree.h
index 5de0508d002..f683fc610af 100644
--- a/lib/libsqlite3/ext/rtree/sqlite3rtree.h
+++ b/lib/libsqlite3/ext/rtree/sqlite3rtree.h
@@ -98,6 +98,8 @@ struct sqlite3_rtree_query_info {
int eParentWithin; /* Visibility of parent node */
int eWithin; /* OUT: Visiblity */
sqlite3_rtree_dbl rScore; /* OUT: Write the score here */
+ /* The following fields are only available in 3.8.11 and later */
+ sqlite3_value **apSqlParam; /* Original SQL values of parameters */
};
/*
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.