Use a different indexer version when external tokenizer is enabled

4 files changed, 23 insertions, 13 deletions

diff --git a/bin/indexer.php b/bin/indexer.php
index 48e98b571..497c6146a 100755
--- a/bin/indexer.php
+++ b/bin/indexer.php
@@ -13,10 +13,6 @@ require_once(DOKU_INC.'inc/auth.php');
 require_once(DOKU_INC.'inc/cliopts.php');
 session_write_close();
 
-// Version tag used to force rebuild on upgrade
-// Need to keep in sync with lib/exe/indexer.php
-if(!defined('INDEXER_VERSION')) define('INDEXER_VERSION', 2);
-
 // handle options
 $short_opts = 'hcuq';
 $long_opts  = array('help', 'clear', 'update', 'quiet');
@@ -88,7 +84,7 @@ function _index($id){
     if(!$CLEAR){
         $idxtag = metaFN($id,'.indexed');
         if(@file_exists($idxtag)){
-            if(io_readFile($idxtag) >= INDEXER_VERSION){
+            if(io_readFile($idxtag) == idx_get_version()){
                 $last = @filemtime(metaFN($id,'.indexed'));
                 if($last > @filemtime(wikiFN($id))) return;
             }
@@ -98,7 +94,7 @@ function _index($id){
     _lock();
     _quietecho("$id... ");
     idx_addPage($id);
-    io_saveFile(metaFN($id,'.indexed'),INDEXER_VERSION);
+    io_saveFile(metaFN($id,'.indexed'), idx_get_version());
     _quietecho("done.\n");
     _unlock();
 }
diff --git a/inc/indexer.php b/inc/indexer.php
index 1c955a99d..4914c9fc6 100644
--- a/inc/indexer.php
+++ b/inc/indexer.php
@@ -8,6 +8,9 @@
 
 if(!defined('DOKU_INC')) die('meh.');
 
+// Version tag used to force rebuild on upgrade
+define('INDEXER_VERSION', 2);
+
 // set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
 if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
 
@@ -43,6 +46,20 @@ define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two charact
 define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
 
 /**
+ * Version of the indexer taking into consideration the external tokenizer.
+ * The indexer is only compatible with data written by the same version.
+ *
+ * @author Tom N Harris <tnharris@whoopdedo.org>
+ */
+function idx_get_version(){
+    global $conf;
+    if($conf['external_tokenizer'])
+        return INDEXER_VERSION . '+' . trim($conf['tokenizer_cmd']);
+    else
+        return INDEXER_VERSION;
+}
+
+/**
  * Measure the length of a string.
  * Differs from strlen in handling of asian characters.
  *
diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php
index 4a6f74ba4..55d860296 100644
--- a/lib/exe/indexer.php
+++ b/lib/exe/indexer.php
@@ -11,9 +11,6 @@ require_once(DOKU_INC.'inc/init.php');
 session_write_close();  //close session
 if(!defined('NL')) define('NL',"\n");
 
-// Version tag used to force rebuild on upgrade
-define('INDEXER_VERSION', 2);
-
 // keep running after browser closes connection
 @ignore_user_abort(true);
 
@@ -140,7 +137,7 @@ function runIndexer(){
     // check if indexing needed
     $idxtag = metaFN($ID,'.indexed');
     if(@file_exists($idxtag)){
-        if(trim(io_readFile($idxtag)) == INDEXER_VERSION){
+        if(trim(io_readFile($idxtag)) == idx_get_version()){
             $last = @filemtime($idxtag);
             if($last > @filemtime(wikiFN($ID))){
                 print "runIndexer(): index for $ID up to date".NL;
@@ -168,7 +165,7 @@ function runIndexer(){
     idx_addPage($ID);
 
     // we're finished - save and free lock
-    io_saveFile(metaFN($ID,'.indexed'),INDEXER_VERSION);
+    io_saveFile(metaFN($ID,'.indexed'), idx_get_version());
     @rmdir($lock);
     print "runIndexer(): finished".NL;
     return true;
diff --git a/lib/exe/xmlrpc.php b/lib/exe/xmlrpc.php
index f06792361..410d4f6ba 100644
--- a/lib/exe/xmlrpc.php
+++ b/lib/exe/xmlrpc.php
@@ -1,7 +1,7 @@
 <?php
 if(!defined('DOKU_INC')) define('DOKU_INC',dirname(__FILE__).'/../../');
 
-// fix when '<?xml' isn't on the very first line
+// fix when '< ?xml' isn't on the very first line
 if(isset($HTTP_RAW_POST_DATA)) $HTTP_RAW_POST_DATA = trim($HTTP_RAW_POST_DATA);
 
 /**
@@ -570,7 +570,7 @@ class dokuwiki_xmlrpc_server extends IXR_IntrospectionServer {
             idx_addPage($id);
 
             // we're finished - save and free lock
-            io_saveFile(metaFN($id,'.indexed'),INDEXER_VERSION);
+            io_saveFile(metaFN($id,'.indexed'), idx_get_version());
             @rmdir($lock);
         }