From c4f79b71351dd0d96f19f7c5629888d85a814c72 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Wed, 7 Apr 2010 11:31:50 +0200 Subject: Sitemap rewrite --- lib/exe/indexer.php | 98 ++--------------------------------------------------- 1 file changed, 3 insertions(+), 95 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index f8e2f7981..63ad5931f 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -232,88 +232,11 @@ function metaUpdate(){ * @link https://www.google.com/webmasters/sitemaps/docs/en/about.html */ function runSitemapper(){ - global $conf; print "runSitemapper(): started".NL; - if(!$conf['sitemap']) return false; - - if($conf['compression'] == 'bz2' || $conf['compression'] == 'gz'){ - $sitemap = 'sitemap.xml.gz'; - }else{ - $sitemap = 'sitemap.xml'; - } - print "runSitemapper(): using $sitemap".NL; - - if(@file_exists(DOKU_INC.$sitemap)){ - if(!is_writable(DOKU_INC.$sitemap)) return false; - }else{ - if(!is_writable(DOKU_INC)) return false; - } - - if(@filesize(DOKU_INC.$sitemap) && - @filemtime(DOKU_INC.$sitemap) > (time()-($conf['sitemap']*60*60*24))){ - print 'runSitemapper(): Sitemap up to date'.NL; - return false; - } - - $pages = idx_getIndex('page', ''); - print 'runSitemapper(): creating sitemap using '.count($pages).' pages'.NL; - - // build the sitemap - ob_start(); - print ''.NL; - print ''.NL; - foreach($pages as $id){ - $id = trim($id); - $file = wikiFN($id); - - //skip hidden, non existing and restricted files - if(isHiddenPage($id)) continue; - $date = @filemtime($file); - if(!$date) continue; - if(auth_aclcheck($id,'','') < AUTH_READ) continue; - - print ' '.NL; - print ' '.wl($id,'',true).''.NL; - print ' '.date_iso8601($date).''.NL; - print ' '.NL; - } - print ''.NL; - $data = ob_get_contents(); - ob_end_clean(); - - //save the new sitemap - io_saveFile(DOKU_INC.$sitemap,$data); - - //ping search engines... - $http = new DokuHTTPClient(); - $http->timeout = 8; - - //ping google - print 'runSitemapper(): pinging google'.NL; - $url = 'http://www.google.com/webmasters/sitemaps/ping?sitemap='; - $url .= urlencode(DOKU_URL.$sitemap); - $resp = $http->get($url); - if($http->error) print 'runSitemapper(): '.$http->error.NL; - print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL; - - //ping yahoo - print 'runSitemapper(): pinging yahoo'.NL; - $url = 'http://search.yahooapis.com/SiteExplorerService/V1/updateNotification?appid=dokuwiki&url='; - $url .= urlencode(DOKU_URL.$sitemap); - $resp = $http->get($url); - if($http->error) print 'runSitemapper(): '.$http->error.NL; - print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL; - - //ping microsoft - print 'runSitemapper(): pinging microsoft'.NL; - $url = 'http://www.bing.com/webmaster/ping.aspx?siteMap='; - $url .= urlencode(DOKU_URL.$sitemap); - $resp = $http->get($url); - if($http->error) print 'runSitemapper(): '.$http->error.NL; - print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL; - + require_once DOKU_INC.'inc/sitemap.php'; + $result = sitemapGenerate() && sitemapPingSearchEngines(); print 'runSitemapper(): finished'.NL; - return true; + return $result; } /** @@ -408,21 +331,6 @@ function sendDigest() { $_SERVER['REMOTE_USER'] = $olduser; } -/** - * Formats a timestamp as ISO 8601 date - * - * @author - * @link http://www.php.net/manual/en/function.date.php#54072 - */ -function date_iso8601($int_date) { - //$int_date: current date in UNIX timestamp - $date_mod = date('Y-m-d\TH:i:s', $int_date); - $pre_timezone = date('O', $int_date); - $time_zone = substr($pre_timezone, 0, 3).":".substr($pre_timezone, 3, 2); - $date_mod .= $time_zone; - return $date_mod; -} - /** * Just send a 1x1 pixel blank gif to the browser * -- cgit v1.2.3 From 2897eb23759202676f5447a72d7fe5eb68321ce3 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Sat, 26 Jun 2010 13:33:46 +0200 Subject: Transformed the sitemapper into a class This makes it possible to autoload the sitemapper when needed. --- lib/exe/indexer.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 63ad5931f..61cf83acc 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -233,8 +233,7 @@ function metaUpdate(){ */ function runSitemapper(){ print "runSitemapper(): started".NL; - require_once DOKU_INC.'inc/sitemap.php'; - $result = sitemapGenerate() && sitemapPingSearchEngines(); + $result = Sitemapper::generate() && Sitemapper::pingSearchEngines(); print 'runSitemapper(): finished'.NL; return $result; } -- cgit v1.2.3 From 74afac00b33f004aad776496f113476acca8efc5 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Mon, 18 Oct 2010 22:03:03 +0200 Subject: removed deprecated index update function --- lib/exe/indexer.php | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index f8e2f7981..f9cc3158a 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -163,10 +163,6 @@ function runIndexer(){ } if($conf['dperm']) chmod($lock, $conf['dperm']); - // upgrade to version 2 - if (!@file_exists($conf['indexdir'].'/pageword.idx')) - idx_upgradePageWords(); - // do the work idx_addPage($ID); -- cgit v1.2.3 From 88c5fee2871e49eecca1ebfa90a19b1faa1d7350 Mon Sep 17 00:00:00 2001 From: Adrian Lang Date: Mon, 25 Oct 2010 11:36:28 +0200 Subject: Honor allowdebug setting in lib/exe/indexer --- lib/exe/indexer.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index f9cc3158a..3a9673ed6 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -26,7 +26,8 @@ if(!$defer){ $ID = cleanID($_REQUEST['id']); // Catch any possible output (e.g. errors) -if(!isset($_REQUEST['debug'])) ob_start(); +$output = isset($_REQUEST['debug']) && $conf['allowdebug']; +if(!$output) ob_start(); // run one of the jobs $tmp = array(); // No event data @@ -42,7 +43,7 @@ if ($evt->advise_before()) { } if($defer) sendGIF(); -if(!isset($_REQUEST['debug'])) ob_end_clean(); +if(!$output) ob_end_clean(); exit; // -------------------------------------------------------------------- -- cgit v1.2.3 From 1172f8dcef2c8198ddcdaffcdf65a735811d20a3 Mon Sep 17 00:00:00 2001 From: Adrian Lang Date: Sat, 13 Nov 2010 18:20:51 +0100 Subject: Introduce metadata write wrapper p_save_metadata p_purge_metadata now updates the metadata cache and the INFO array like the other metadata writing functions --- lib/exe/indexer.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index f35f9ed72..3fa81715b 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -190,7 +190,7 @@ function metaUpdate(){ // rendering needed? if (@file_exists($file)) return false; - if (!@file_exists(wikiFN($ID))) return false; + if (!page_exists($ID)) return false; global $conf; @@ -213,7 +213,7 @@ function metaUpdate(){ } $meta = p_render_metadata($ID, $meta); - io_saveFile($file, serialize($meta)); + p_save_metadata($ID, $meta); echo "metaUpdate(): finished".NL; return true; -- cgit v1.2.3 From ee0891d8ffd7e4a59c958b9546a3b8382e4e5991 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sun, 14 Nov 2010 14:18:51 -0500 Subject: Do not assume that index files will be backward compatible --- lib/exe/indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 3fa81715b..4a6f74ba4 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -140,7 +140,7 @@ function runIndexer(){ // check if indexing needed $idxtag = metaFN($ID,'.indexed'); if(@file_exists($idxtag)){ - if(io_readFile($idxtag) >= INDEXER_VERSION){ + if(trim(io_readFile($idxtag)) == INDEXER_VERSION){ $last = @filemtime($idxtag); if($last > @filemtime(wikiFN($ID))){ print "runIndexer(): index for $ID up to date".NL; -- cgit v1.2.3 From 7c2ef4e8d524fb9262c5a08831220f9fb2dc11fe Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 17 Nov 2010 17:02:31 -0500 Subject: Use a different indexer version when external tokenizer is enabled --- lib/exe/indexer.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 4a6f74ba4..55d860296 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -11,9 +11,6 @@ require_once(DOKU_INC.'inc/init.php'); session_write_close(); //close session if(!defined('NL')) define('NL',"\n"); -// Version tag used to force rebuild on upgrade -define('INDEXER_VERSION', 2); - // keep running after browser closes connection @ignore_user_abort(true); @@ -140,7 +137,7 @@ function runIndexer(){ // check if indexing needed $idxtag = metaFN($ID,'.indexed'); if(@file_exists($idxtag)){ - if(trim(io_readFile($idxtag)) == INDEXER_VERSION){ + if(trim(io_readFile($idxtag)) == idx_get_version()){ $last = @filemtime($idxtag); if($last > @filemtime(wikiFN($ID))){ print "runIndexer(): index for $ID up to date".NL; @@ -168,7 +165,7 @@ function runIndexer(){ idx_addPage($ID); // we're finished - save and free lock - io_saveFile(metaFN($ID,'.indexed'),INDEXER_VERSION); + io_saveFile(metaFN($ID,'.indexed'), idx_get_version()); @rmdir($lock); print "runIndexer(): finished".NL; return true; -- cgit v1.2.3 From 98214867894eba512bf47cba3439ccba3968f49b Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 22 Nov 2010 21:12:02 +0100 Subject: Render metadata when needed This changes fundamentally when metadata is rendered. This commit introduces a new cache file for every page that just contains a timestamp and is updated whenever the metadata of that page is rendered. Metadata is rendered when p_get_metadata is called and the last rendering has been before a page, metadata, configuration or renderer update or purge is set like in the xhtml renderer cache. Metadata is no longer automatically rendered when the xhtml renderer cache isn't used but will still be rendered when needed as p_get_metadata is called in the cache. Metadata is also no longer rendered in the indexer script when missing as that is already done by pageinfo() before anything else is done so the indexer script won't be called when there is no metadata file. --- lib/exe/indexer.php | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 3fa81715b..bf5bad2e7 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -34,7 +34,6 @@ $tmp = array(); // No event data $evt = new Doku_Event('INDEXER_TASKS_RUN', $tmp); if ($evt->advise_before()) { runIndexer() or - metaUpdate() or runSitemapper() or sendDigest() or runTrimRecentChanges() or @@ -174,51 +173,6 @@ function runIndexer(){ return true; } -/** - * Will render the metadata for the page if not exists yet - * - * This makes sure pages which are created from outside DokuWiki will - * gain their data when viewed for the first time. - */ -function metaUpdate(){ - global $ID; - print "metaUpdate(): started".NL; - - if(!$ID) return false; - $file = metaFN($ID, '.meta'); - echo "meta file: $file".NL; - - // rendering needed? - if (@file_exists($file)) return false; - if (!page_exists($ID)) return false; - - global $conf; - - // gather some additional info from changelog - $info = io_grep($conf['changelog'], - '/^(\d+)\t(\d+\.\d+\.\d+\.\d+)\t'.preg_quote($ID,'/').'\t([^\t]+)\t([^\t\n]+)/', - 0,true); - - $meta = array(); - if(!empty($info)){ - $meta['date']['created'] = $info[0][1]; - foreach($info as $item){ - if($item[4] != '*'){ - $meta['date']['modified'] = $item[1]; - if($item[3]){ - $meta['contributor'][$item[3]] = $item[3]; - } - } - } - } - - $meta = p_render_metadata($ID, $meta); - p_save_metadata($ID, $meta); - - echo "metaUpdate(): finished".NL; - return true; -} - /** * Builds a Google Sitemap of all public pages known to the indexer * -- cgit v1.2.3 From e3776c06c37cc197709dac60892604dfea894ac2 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 29 Nov 2010 01:34:36 +0100 Subject: Remove enc=utf-8 in VIM modeline as it is not allowed in VIM 7.3 As of VIM 7.3 it is no longer possible to specify the encoding in the modeline. This gives an error message whenever such a file is opened, thus this commit removes the enc setting from the modeline. --- lib/exe/indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index bf5bad2e7..58b0d0787 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -302,6 +302,6 @@ function sendGIF(){ // Thinks it's got the whole image } -//Setup VIM: ex: et ts=4 enc=utf-8 : +//Setup VIM: ex: et ts=4 : // No trailing PHP closing tag - no output please! // See Note at http://www.php.net/manual/en/language.basic-syntax.instruction-separation.php -- cgit v1.2.3 From 9b41be2446ea725a496f34b28ac4db84bece57c9 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 29 Dec 2010 03:50:05 -0500 Subject: Indexer v3 Rewrite part two, update uses of indexer --- lib/exe/indexer.php | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 55d860296..a5a7d6b2a 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -134,41 +134,8 @@ function runIndexer(){ if(!$ID) return false; - // check if indexing needed - $idxtag = metaFN($ID,'.indexed'); - if(@file_exists($idxtag)){ - if(trim(io_readFile($idxtag)) == idx_get_version()){ - $last = @filemtime($idxtag); - if($last > @filemtime(wikiFN($ID))){ - print "runIndexer(): index for $ID up to date".NL; - return false; - } - } - } - - // try to aquire a lock - $lock = $conf['lockdir'].'/_indexer.lock'; - while(!@mkdir($lock,$conf['dmode'])){ - usleep(50); - if(time()-@filemtime($lock) > 60*5){ - // looks like a stale lock - remove it - @rmdir($lock); - print "runIndexer(): stale lock removed".NL; - }else{ - print "runIndexer(): indexer locked".NL; - return false; - } - } - if($conf['dperm']) chmod($lock, $conf['dperm']); - // do the work - idx_addPage($ID); - - // we're finished - save and free lock - io_saveFile(metaFN($ID,'.indexed'), idx_get_version()); - @rmdir($lock); - print "runIndexer(): finished".NL; - return true; + return idx_addPage($ID, true); } /** -- cgit v1.2.3 From 1b052f5cb72bee256af579602cbbed59492b2759 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sun, 16 Jan 2011 19:29:03 +0100 Subject: increase indexer version to reforce rebuild for the new title index --- lib/exe/indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/exe/indexer.php') diff --git a/lib/exe/indexer.php b/lib/exe/indexer.php index 58b0d0787..eec8c968c 100644 --- a/lib/exe/indexer.php +++ b/lib/exe/indexer.php @@ -12,7 +12,7 @@ session_write_close(); //close session if(!defined('NL')) define('NL',"\n"); // Version tag used to force rebuild on upgrade -define('INDEXER_VERSION', 2); +define('INDEXER_VERSION', 3); // keep running after browser closes connection @ignore_user_abort(true); -- cgit v1.2.3