Merge branch 'tokenizer-rewrite' into michitux

4 files changed, 8 insertions, 7 deletions

diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php
index 3fa81715b..55d860296 100644
--- a/lib/exe/indexer.php
+++ b/lib/exe/indexer.php
@@ -11,9 +11,6 @@ require_once(DOKU_INC.'inc/init.php');
 session_write_close();  //close session
 if(!defined('NL')) define('NL',"\n");
 
-// Version tag used to force rebuild on upgrade
-define('INDEXER_VERSION', 2);
-
 // keep running after browser closes connection
 @ignore_user_abort(true);
 
@@ -140,7 +137,7 @@ function runIndexer(){
     // check if indexing needed
     $idxtag = metaFN($ID,'.indexed');
     if(@file_exists($idxtag)){
-        if(io_readFile($idxtag) >= INDEXER_VERSION){
+        if(trim(io_readFile($idxtag)) == idx_get_version()){
             $last = @filemtime($idxtag);
             if($last > @filemtime(wikiFN($ID))){
                 print "runIndexer(): index for $ID up to date".NL;
@@ -168,7 +165,7 @@ function runIndexer(){
     idx_addPage($ID);
 
     // we're finished - save and free lock
-    io_saveFile(metaFN($ID,'.indexed'),INDEXER_VERSION);
+    io_saveFile(metaFN($ID,'.indexed'), idx_get_version());
     @rmdir($lock);
     print "runIndexer(): finished".NL;
     return true;
diff --git a/lib/exe/xmlrpc.php b/lib/exe/xmlrpc.php
index f06792361..410d4f6ba 100644
--- a/lib/exe/xmlrpc.php
+++ b/lib/exe/xmlrpc.php
@@ -1,7 +1,7 @@
 <?php
 if(!defined('DOKU_INC')) define('DOKU_INC',dirname(__FILE__).'/../../');
 
-// fix when '<?xml' isn't on the very first line
+// fix when '< ?xml' isn't on the very first line
 if(isset($HTTP_RAW_POST_DATA)) $HTTP_RAW_POST_DATA = trim($HTTP_RAW_POST_DATA);
 
 /**
@@ -570,7 +570,7 @@ class dokuwiki_xmlrpc_server extends IXR_IntrospectionServer {
             idx_addPage($id);
 
             // we're finished - save and free lock
-            io_saveFile(metaFN($id,'.indexed'),INDEXER_VERSION);
+            io_saveFile(metaFN($id,'.indexed'), idx_get_version());
             @rmdir($lock);
         }
 
diff --git a/lib/plugins/config/lang/en/lang.php b/lib/plugins/config/lang/en/lang.php
index a944d6bd7..85214bf98 100644
--- a/lib/plugins/config/lang/en/lang.php
+++ b/lib/plugins/config/lang/en/lang.php
@@ -141,6 +141,8 @@ $lang['renderer_xhtml']   = 'Renderer to use for main (xhtml) wiki output';
 $lang['renderer__core']   = '%s (dokuwiki core)';
 $lang['renderer__plugin'] = '%s (plugin)';
 $lang['rememberme'] = 'Allow permanent login cookies (remember me)';
+$lang['external_tokenizer'] = 'Use an external program to split pages into words for indexing';
+$lang['tokenizer_cmd'] = 'Command line to start the external tokenizer';
 
 $lang['rss_type']    = 'XML feed type';
 $lang['rss_linkto']  = 'XML feed links to';
diff --git a/lib/plugins/config/settings/config.metadata.php b/lib/plugins/config/settings/config.metadata.php
index edba65262..331da5ab8 100644
--- a/lib/plugins/config/settings/config.metadata.php
+++ b/lib/plugins/config/settings/config.metadata.php
@@ -190,6 +190,8 @@ $meta['broken_iua']  = array('onoff');
 $meta['xsendfile']   = array('multichoice','_choices' => array(0,1,2,3));
 $meta['renderer_xhtml'] = array('renderer','_format' => 'xhtml','_choices' => array('xhtml'));
 $meta['readdircache'] = array('numeric');
+$meta['external_tokenizer'] = array('onoff');
+$meta['tokenizer_cmd'] = array('string');
 
 $meta['_network']    = array('fieldset');
 $meta['proxy____host'] = array('string','_pattern' => '#^(|[a-z0-9\-\.+]+)$#i');