From 74afac00b33f004aad776496f113476acca8efc5 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Mon, 18 Oct 2010 22:03:03 +0200 Subject: removed deprecated index update function --- bin/indexer.php | 7 ------- 1 file changed, 7 deletions(-) (limited to 'bin') diff --git a/bin/indexer.php b/bin/indexer.php index c95314d7c..48e98b571 100755 --- a/bin/indexer.php +++ b/bin/indexer.php @@ -71,13 +71,6 @@ function _usage() { function _update(){ global $conf; - // upgrade to version 2 - if (!@file_exists($conf['indexdir'].'/pageword.idx')){ - _lock(); - idx_upgradePageWords(); - _unlock(); - } - $data = array(); _quietecho("Searching pages... "); search($data,$conf['datadir'],'search_allpages',array('skipacl' => true)); -- cgit v1.2.3 From 7c2ef4e8d524fb9262c5a08831220f9fb2dc11fe Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 17 Nov 2010 17:02:31 -0500 Subject: Use a different indexer version when external tokenizer is enabled --- bin/indexer.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'bin') diff --git a/bin/indexer.php b/bin/indexer.php index 48e98b571..497c6146a 100755 --- a/bin/indexer.php +++ b/bin/indexer.php @@ -13,10 +13,6 @@ require_once(DOKU_INC.'inc/auth.php'); require_once(DOKU_INC.'inc/cliopts.php'); session_write_close(); -// Version tag used to force rebuild on upgrade -// Need to keep in sync with lib/exe/indexer.php -if(!defined('INDEXER_VERSION')) define('INDEXER_VERSION', 2); - // handle options $short_opts = 'hcuq'; $long_opts = array('help', 'clear', 'update', 'quiet'); @@ -88,7 +84,7 @@ function _index($id){ if(!$CLEAR){ $idxtag = metaFN($id,'.indexed'); if(@file_exists($idxtag)){ - if(io_readFile($idxtag) >= INDEXER_VERSION){ + if(io_readFile($idxtag) == idx_get_version()){ $last = @filemtime(metaFN($id,'.indexed')); if($last > @filemtime(wikiFN($id))) return; } @@ -98,7 +94,7 @@ function _index($id){ _lock(); _quietecho("$id... "); idx_addPage($id); - io_saveFile(metaFN($id,'.indexed'),INDEXER_VERSION); + io_saveFile(metaFN($id,'.indexed'), idx_get_version()); _quietecho("done.\n"); _unlock(); } -- cgit v1.2.3 From e3776c06c37cc197709dac60892604dfea894ac2 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 29 Nov 2010 01:34:36 +0100 Subject: Remove enc=utf-8 in VIM modeline as it is not allowed in VIM 7.3 As of VIM 7.3 it is no longer possible to specify the encoding in the modeline. This gives an error message whenever such a file is opened, thus this commit removes the enc setting from the modeline. --- bin/indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/indexer.php b/bin/indexer.php index 48e98b571..67254fa88 100755 --- a/bin/indexer.php +++ b/bin/indexer.php @@ -163,4 +163,4 @@ function _quietecho($msg) { if(!$QUIET) echo $msg; } -//Setup VIM: ex: et ts=2 enc=utf-8 : +//Setup VIM: ex: et ts=2 : -- cgit v1.2.3 From c0470665eec5d52d0e7da72e5ba8fd76b931247d Mon Sep 17 00:00:00 2001 From: "Martin 'E.T.' Misuth" Date: Tue, 5 Oct 2010 17:01:13 +0200 Subject: Added striplangs.php script, with which you can strip out unused languages out of DW to make it smaller (eg. FTP upload) --- bin/striplangs.php | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 bin/striplangs.php (limited to 'bin') diff --git a/bin/striplangs.php b/bin/striplangs.php new file mode 100644 index 000000000..a53fe580f --- /dev/null +++ b/bin/striplangs.php @@ -0,0 +1,143 @@ +#!/usr/bin/php +get($short); + if ( is_null($arg) ) { + $arg = $OPTS->get($long); + } + return $arg; +} + +function processPlugins($path, $keep_langs) { + if (is_dir($path)) { + $entries = scandir($path); + + foreach ($entries as $entry) { + if ($entry != "." && $entry != "..") { + if ( is_dir($path.'\\'.$entry) ) { + + $plugin_langs = $path.'\\'.$entry.'\\lang'; + + if ( is_dir( $plugin_langs ) ) { + stripDirLangs($plugin_langs, $keep_langs); + } + } + } + } + } +} + +function stripDirLangs($path, $keep_langs) { + $dir = dir($path); + + while(($cur_dir = $dir->read()) !== false) { + if( $cur_dir != '.' and $cur_dir != '..' and is_dir($path.'\\'.$cur_dir)) { + + if ( !in_array($cur_dir, $keep_langs, true ) ) { + killDir($path.'\\'.$cur_dir); + } + } + } + $dir->close(); +} + +function killDir($dir) { + if (is_dir($dir)) { + $entries = scandir($dir); + + foreach ($entries as $entry) { + if ($entry != "." && $entry != "..") { + if ( is_dir($dir.'\\'.$entry) ) { + killDir($dir.'\\'.$entry); + } else { + unlink($dir.'\\'.$entry); + } + } + } + reset($entries); + rmdir($dir); + } +} +#------------------------------------------------------------------------------ + +// handle options +$short_opts = 'hxk:e'; +$long_opts = array('help', 'examples', 'keep=','english'); + +$OPTS = Doku_Cli_Opts::getOptions(__FILE__, $short_opts, $long_opts); + +if ( $OPTS->isError() ) { + fwrite( STDERR, $OPTS->getMessage() . "\n"); + exit(1); +} + +// handle '--examples' option +$show_examples = ( $OPTS->has('x') or $OPTS->has('examples') ) ? true : false; + +// handle '--help' option +if ( $OPTS->has('h') or $OPTS->has('help') ) { + usage($show_examples); + exit(0); +} + +// handle both '--keep' and '--english' options +if ( $OPTS->has('k') or $OPTS->has('keep') ) { + $preserved_langs = getSuppliedArgument($OPTS,'k','keep'); + $langs = explode(',', $preserved_langs); + + // ! always enforce 'en' lang when using '--keep' (DW relies on it) + if ( !isset($langs['en']) ) { + $langs[]='en'; + } +} elseif ( $OPTS->has('e') or $OPTS->has('english') ) { + // '--english' was specified strip everything besides 'en' + $langs = array ('en'); +} else { + // no option was specified, print usage but don't do anything as + // this run might not be intented + usage(); + print "\n + ERROR + No option specified, use either -h -x to get more info, + or -e to strip every language besides english.\n"; + exit(1); +} + +// Kill all language directories in /inc/lang and /lib/plugins besides those in $langs array +stripDirLangs(realpath(dirname(__FILE__).'/../inc/lang'), $langs); +processPlugins(realpath(dirname(__FILE__).'/../lib/plugins'), $langs); \ No newline at end of file -- cgit v1.2.3 From 79e197ef89cfb21cb763883243e6338d92ab28c7 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Mon, 29 Nov 2010 20:41:17 +0100 Subject: use forward slashes as path separator in striplangs.php --- bin/striplangs.php | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'bin') diff --git a/bin/striplangs.php b/bin/striplangs.php index a53fe580f..288859c6a 100644 --- a/bin/striplangs.php +++ b/bin/striplangs.php @@ -1,5 +1,10 @@ #!/usr/bin/php + */ if ('cli' != php_sapi_name()) die(); #------------------------------------------------------------------------------ @@ -16,20 +21,20 @@ function usage($show_examples = false) { OPTIONS -h, --help get this help -x, --examples get also usage examples - -k, --keep comma separated list of languages, -e is always implied + -k, --keep comma separated list of languages, -e is always implied -e, --english keeps english, dummy to use without -k"; if ( $show_examples ) { print "\n EXAMPLES - Strips all languages, but keeps 'en' and 'de': + Strips all languages, but keeps 'en' and 'de': striplangs -k de Strips all but 'en','ca-valencia','cs','de','is','sk': striplangs --keep ca-valencia,cs,de,is,sk - + Strips all but 'en': striplangs -e - + No option specified, prints usage and throws error: striplangs\n"; } @@ -49,16 +54,16 @@ function processPlugins($path, $keep_langs) { foreach ($entries as $entry) { if ($entry != "." && $entry != "..") { - if ( is_dir($path.'\\'.$entry) ) { - - $plugin_langs = $path.'\\'.$entry.'\\lang'; - + if ( is_dir($path.'/'.$entry) ) { + + $plugin_langs = $path.'/'.$entry.'/lang'; + if ( is_dir( $plugin_langs ) ) { stripDirLangs($plugin_langs, $keep_langs); } } } - } + } } } @@ -66,15 +71,15 @@ function stripDirLangs($path, $keep_langs) { $dir = dir($path); while(($cur_dir = $dir->read()) !== false) { - if( $cur_dir != '.' and $cur_dir != '..' and is_dir($path.'\\'.$cur_dir)) { + if( $cur_dir != '.' and $cur_dir != '..' and is_dir($path.'/'.$cur_dir)) { if ( !in_array($cur_dir, $keep_langs, true ) ) { - killDir($path.'\\'.$cur_dir); + killDir($path.'/'.$cur_dir); } } } $dir->close(); -} +} function killDir($dir) { if (is_dir($dir)) { @@ -82,13 +87,13 @@ function killDir($dir) { foreach ($entries as $entry) { if ($entry != "." && $entry != "..") { - if ( is_dir($dir.'\\'.$entry) ) { - killDir($dir.'\\'.$entry); + if ( is_dir($dir.'/'.$entry) ) { + killDir($dir.'/'.$entry); } else { - unlink($dir.'\\'.$entry); + unlink($dir.'/'.$entry); } } - } + } reset($entries); rmdir($dir); } @@ -119,11 +124,11 @@ if ( $OPTS->has('h') or $OPTS->has('help') ) { if ( $OPTS->has('k') or $OPTS->has('keep') ) { $preserved_langs = getSuppliedArgument($OPTS,'k','keep'); $langs = explode(',', $preserved_langs); - + // ! always enforce 'en' lang when using '--keep' (DW relies on it) if ( !isset($langs['en']) ) { $langs[]='en'; - } + } } elseif ( $OPTS->has('e') or $OPTS->has('english') ) { // '--english' was specified strip everything besides 'en' $langs = array ('en'); @@ -140,4 +145,4 @@ if ( $OPTS->has('k') or $OPTS->has('keep') ) { // Kill all language directories in /inc/lang and /lib/plugins besides those in $langs array stripDirLangs(realpath(dirname(__FILE__).'/../inc/lang'), $langs); -processPlugins(realpath(dirname(__FILE__).'/../lib/plugins'), $langs); \ No newline at end of file +processPlugins(realpath(dirname(__FILE__).'/../lib/plugins'), $langs); -- cgit v1.2.3 From 9b41be2446ea725a496f34b28ac4db84bece57c9 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 29 Dec 2010 03:50:05 -0500 Subject: Indexer v3 Rewrite part two, update uses of indexer --- bin/indexer.php | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) (limited to 'bin') diff --git a/bin/indexer.php b/bin/indexer.php index 497c6146a..0d523df6e 100755 --- a/bin/indexer.php +++ b/bin/indexer.php @@ -24,6 +24,7 @@ if ( $OPTS->isError() ) { } $CLEAR = false; $QUIET = false; +$INDEXER = null; foreach ($OPTS->options as $key => $val) { switch ($key) { case 'h': @@ -66,6 +67,9 @@ function _usage() { function _update(){ global $conf; + global $INDEXER; + + $INDEXER = idx_get_indexer(); $data = array(); _quietecho("Searching pages... "); @@ -78,25 +82,47 @@ function _update(){ } function _index($id){ + global $INDEXER; global $CLEAR; + global $QUIET; // if not cleared only update changed and new files if(!$CLEAR){ $idxtag = metaFN($id,'.indexed'); if(@file_exists($idxtag)){ if(io_readFile($idxtag) == idx_get_version()){ - $last = @filemtime(metaFN($id,'.indexed')); + $last = @filemtime($idxtag); if($last > @filemtime(wikiFN($id))) return; } } } - _lock(); _quietecho("$id... "); - idx_addPage($id); - io_saveFile(metaFN($id,'.indexed'), idx_get_version()); + $body = ''; + $data = array($id, $body); + $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); + if ($evt->advise_before()) $data[1] = $data[1] . " " . rawWiki($id); + $evt->advise_after(); + unset($evt); + list($id,$body) = $data; + $said = false; + while(true) { + $result = $INDEXER->addPageWords($id, $body); + if ($result == "locked") { + if($said){ + _quietecho("."); + }else{ + _quietecho("Waiting for lockfile (max. 5 min)"); + $said = true; + } + sleep(15); + } else { + break; + } + } + if ($result) + io_saveFile(metaFN($id,'.indexed'), idx_get_version()); _quietecho("done.\n"); - _unlock(); } /** @@ -141,7 +167,7 @@ function _clearindex(){ _lock(); _quietecho("Clearing index... "); io_saveFile($conf['indexdir'].'/page.idx',''); - io_saveFile($conf['indexdir'].'/title.idx',''); + //io_saveFile($conf['indexdir'].'/title.idx',''); $dir = @opendir($conf['indexdir']); if($dir!==false){ while(($f = readdir($dir)) !== false){ @@ -150,6 +176,7 @@ function _clearindex(){ @unlink($conf['indexdir']."/$f"); } } + @unlink($conf['indexdir'].'/lengths.idx'); _quietecho("done.\n"); _unlock(); } -- cgit v1.2.3 From ad79cb7c93a655f864c633433e743b03685b5719 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Sun, 6 Mar 2011 14:48:58 +0100 Subject: Adjust bin/indexer.php for the new indexer Now the indexer is directly called instead of duplicating a large part of the indexer code. --- bin/indexer.php | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) (limited to 'bin') diff --git a/bin/indexer.php b/bin/indexer.php index 85e990bbe..6ee0a9e8d 100755 --- a/bin/indexer.php +++ b/bin/indexer.php @@ -87,41 +87,15 @@ function _index($id){ global $QUIET; // if not cleared only update changed and new files - if(!$CLEAR){ + if($CLEAR){ $idxtag = metaFN($id,'.indexed'); if(@file_exists($idxtag)){ - if(io_readFile($idxtag) == idx_get_version()){ - $last = @filemtime($idxtag); - if($last > @filemtime(wikiFN($id))) return; - } + @unlink($idxtag); } } _quietecho("$id... "); - $body = ''; - $data = array($id, $body); - $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); - if ($evt->advise_before()) $data[1] = $data[1] . " " . rawWiki($id); - $evt->advise_after(); - unset($evt); - list($id,$body) = $data; - $said = false; - while(true) { - $result = $INDEXER->addPageWords($id, $body); - if ($result == "locked") { - if($said){ - _quietecho("."); - }else{ - _quietecho("Waiting for lockfile (max. 5 min)"); - $said = true; - } - sleep(15); - } else { - break; - } - } - if ($result) - io_saveFile(metaFN($id,'.indexed'), idx_get_version()); + idx_addPage($id, !$QUIET); _quietecho("done.\n"); } -- cgit v1.2.3