From 3903be5dc10c8ce0270ce28c57a5b76df87db4c3 Mon Sep 17 00:00:00 2001 From: Adrian Lang Date: Sat, 13 Nov 2010 16:25:43 +0100 Subject: Remove metadata conversion from 0a7e3bce (2006-11-26) --- inc/parserutils.php | 23 ----------------------- 1 file changed, 23 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index 27a5190bd..847b0382f 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -337,9 +337,6 @@ function p_purge_metadata($id) { * read the metadata from source/cache for $id * (internal use only - called by p_get_metadata & p_set_metadata) * - * this function also converts the metadata from the original format to - * the current format ('current' & 'persistent' arrays) - * * @author Christopher Smith * * @param string $id absolute wiki page id @@ -356,26 +353,6 @@ function p_read_metadata($id,$cache=false) { $file = metaFN($id, '.meta'); $meta = @file_exists($file) ? unserialize(io_readFile($file, false)) : array('current'=>array(),'persistent'=>array()); - // convert $meta from old format to new (current+persistent) format - if (!isset($meta['current'])) { - $meta = array('current'=>$meta,'persistent'=>$meta); - - // remove non-persistent keys - unset($meta['persistent']['title']); - unset($meta['persistent']['description']['abstract']); - unset($meta['persistent']['description']['tableofcontents']); - unset($meta['persistent']['relation']['haspart']); - unset($meta['persistent']['relation']['references']); - unset($meta['persistent']['date']['valid']); - - if (empty($meta['persistent']['description'])) unset($meta['persistent']['description']); - if (empty($meta['persistent']['relation'])) unset($meta['persistent']['relation']); - if (empty($meta['persistent']['date'])) unset($meta['persistent']['date']); - - // save converted metadata - io_saveFile($file, serialize($meta)); - } - if ($cache) { $cache_metadata[(string)$id] = $meta; } -- cgit v1.2.3 From 1172f8dcef2c8198ddcdaffcdf65a735811d20a3 Mon Sep 17 00:00:00 2001 From: Adrian Lang Date: Sat, 13 Nov 2010 18:20:51 +0100 Subject: Introduce metadata write wrapper p_save_metadata p_purge_metadata now updates the metadata cache and the INFO array like the other metadata writing functions --- inc/parserutils.php | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index 847b0382f..b8b063fc3 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -223,7 +223,7 @@ function p_get_instructions($text){ * @author Esther Brunner */ function p_get_metadata($id, $key='', $render=false){ - global $ID, $INFO, $cache_metadata; + global $ID; // cache the current page // Benchmarking shows the current page's metadata is generally the only page metadata @@ -234,11 +234,7 @@ function p_get_metadata($id, $key='', $render=false){ // metadata has never been rendered before - do it! (but not for non-existent pages) if ($render && !isset($meta['current']['description']['abstract']) && page_exists($id)){ $meta = p_render_metadata($id, $meta); - io_saveFile(metaFN($id, '.meta'), serialize($meta)); - - // sync cached copies, including $INFO metadata - if (!empty($cache_metadata[$id])) $cache_metadata[$id] = $meta; - if (!empty($INFO) && ($id == $INFO['id'])) { $INFO['meta'] = $meta['current']; } + p_save_metadata($id, $meta); } $val = $meta['current']; @@ -305,13 +301,7 @@ function p_set_metadata($id, $data, $render=false, $persistent=true){ // save only if metadata changed if ($meta == $orig) return true; - // sync cached copies, including $INFO metadata - global $cache_metadata, $INFO; - - if (!empty($cache_metadata[$id])) $cache_metadata[$id] = $meta; - if (!empty($INFO) && ($id == $INFO['id'])) { $INFO['meta'] = $meta['current']; } - - return io_saveFile(metaFN($id, '.meta'), serialize($meta)); + return p_save_metadata($id, $meta); } /** @@ -321,16 +311,16 @@ function p_set_metadata($id, $data, $render=false, $persistent=true){ * @author Michael Klier */ function p_purge_metadata($id) { - $metafn = metaFN('id', '.meta'); - $meta = p_read_metadata($id); + $meta = p_read_metadata($id); foreach($meta['current'] as $key => $value) { if(is_array($meta[$key])) { $meta['current'][$key] = array(); } else { $meta['current'][$key] = ''; } + } - return io_saveFile(metaFN($id, '.meta'), serialize($meta)); + return p_save_metadata($id, $meta); } /** @@ -360,6 +350,24 @@ function p_read_metadata($id,$cache=false) { return $meta; } +/** + * This is the backend function to save a metadata array to a file + * + * @param string $id absolute wiki page id + * @param array $meta metadata + * + * @return bool success / fail + */ +function p_save_metadata($id, $meta) { + // sync cached copies, including $INFO metadata + global $cache_metadata, $INFO; + + if (isset($cache_metadata[$id])) $cache_metadata[$id] = $meta; + if (!empty($INFO) && ($id == $INFO['id'])) { $INFO['meta'] = $meta['current']; } + + return io_saveFile(metaFN($id, '.meta'), serialize($meta)); +} + /** * renders the metadata of a page * -- cgit v1.2.3 From a365baeef4fc0b6d593043c6db53c01671de9490 Mon Sep 17 00:00:00 2001 From: Dominik Eckelmann Date: Sat, 13 Nov 2010 19:04:26 +0100 Subject: improved some metadata comments --- inc/parserutils.php | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index b8b063fc3..a50e3f4f3 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -252,6 +252,15 @@ function p_get_metadata($id, $key='', $render=false){ /** * sets metadata elements of a page * + * @see http://www.dokuwiki.org/devel:metadata#functions_to_get_and_set_metadata + * + * @param String $id is the ID of a wiki page + * @param Array $data is an array with key ⇒ value pairs to be set in the metadata + * @param Boolean $render whether or not the page metadata should be generated with the renderer + * @param Boolean $persistent indicates whether or not the particular metadata value will persist through + * the next metadata rendering. + * @return boolean true on success + * * @author Esther Brunner */ function p_set_metadata($id, $data, $render=false, $persistent=true){ -- cgit v1.2.3 From 98214867894eba512bf47cba3439ccba3968f49b Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 22 Nov 2010 21:12:02 +0100 Subject: Render metadata when needed This changes fundamentally when metadata is rendered. This commit introduces a new cache file for every page that just contains a timestamp and is updated whenever the metadata of that page is rendered. Metadata is rendered when p_get_metadata is called and the last rendering has been before a page, metadata, configuration or renderer update or purge is set like in the xhtml renderer cache. Metadata is no longer automatically rendered when the xhtml renderer cache isn't used but will still be rendered when needed as p_get_metadata is called in the cache. Metadata is also no longer rendered in the indexer script when missing as that is already done by pageinfo() before anything else is done so the indexer script won't be called when there is no metadata file. --- inc/parserutils.php | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index a50e3f4f3..d4f55a6e4 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -221,6 +221,7 @@ function p_get_instructions($text){ * returns the metadata of a page * * @author Esther Brunner + * @author Michael Hamann */ function p_get_metadata($id, $key='', $render=false){ global $ID; @@ -231,10 +232,24 @@ function p_get_metadata($id, $key='', $render=false){ $cache = ($ID == $id); $meta = p_read_metadata($id, $cache); - // metadata has never been rendered before - do it! (but not for non-existent pages) - if ($render && !isset($meta['current']['description']['abstract']) && page_exists($id)){ - $meta = p_render_metadata($id, $meta); - p_save_metadata($id, $meta); + // prevent recursive calls in the cache + static $recursion = false; + if (!$recursion){ + $recursion = true; + + $cachefile = new cache_renderer($id, wikiFN($id), 'metadata'); + + if (page_exists($id) && !$cachefile->useCache()){ + $meta = p_render_metadata($id, $meta); + if (p_save_metadata($id, $meta)) { + // store a timestamp in order to make sure that the cachefile is touched + $cachefile->storeCache(time()); + } else { + msg('Unable to save metadata file. Hint: disk full; file permissions; safe_mode setting.',-1); + } + } + + $recursion = false; } $val = $meta['current']; -- cgit v1.2.3 From 69ba640bb6c63d2132ca401588053dfc507dbb1b Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 27 Dec 2010 22:51:06 +0100 Subject: Save metadata only when really changed This avoids disk writes when not needed and possibly also xhtml rendering when the metadata needs to be rendered but xhtml doesn't (unless the metadata file is changed). --- inc/parserutils.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index d4f55a6e4..fbdc2e3a9 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -240,8 +240,10 @@ function p_get_metadata($id, $key='', $render=false){ $cachefile = new cache_renderer($id, wikiFN($id), 'metadata'); if (page_exists($id) && !$cachefile->useCache()){ + $old_meta = $meta; $meta = p_render_metadata($id, $meta); - if (p_save_metadata($id, $meta)) { + // only update the file when the metadata has been changed + if ($meta == $old_meta || p_save_metadata($id, $meta)) { // store a timestamp in order to make sure that the cachefile is touched $cachefile->storeCache(time()); } else { -- cgit v1.2.3 From 4a81940267e4278153d3726b605286fd963084ec Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 10 Jan 2011 20:39:44 +0100 Subject: Activate the render parameter of p_get_metadata p_get_metadata has a $render parameter that has been disabled by the restructuring of metadata rendering. This change reactivates it so rendering metadata can be prevented. This is e.g. used in the search and in some plugins like indexmenu that use p_get_first_heading. The default of the parameter has been changed to true as otherwise the new caching structure won't work as almost all calls to p_get_metadata don't set the $render parameter. The indexer call to p_get_first_heading has been changed to set $render to true as in the indexer only one page will be rendered and the title in the index should really be the current one. This does not fix the problem that rendering pages with lots of links or displaying the index can cause the parsing/rendering of a lot of pages. --- inc/parserutils.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index fbdc2e3a9..9224dae10 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -220,10 +220,15 @@ function p_get_instructions($text){ /** * returns the metadata of a page * + * @param string $id The id of the page the metadata should be returned from + * @param string $key The key of the metdata value that shall be read (by default everything) - separate hierarchies by " " like "date created" + * @param boolean $render If the page should be rendererd when the cache can't be used - default true + * @return mixed The requested metadata fields + * * @author Esther Brunner * @author Michael Hamann */ -function p_get_metadata($id, $key='', $render=false){ +function p_get_metadata($id, $key='', $render=true){ global $ID; // cache the current page @@ -234,7 +239,7 @@ function p_get_metadata($id, $key='', $render=false){ // prevent recursive calls in the cache static $recursion = false; - if (!$recursion){ + if (!$recursion && $render){ $recursion = true; $cachefile = new cache_renderer($id, wikiFN($id), 'metadata'); -- cgit v1.2.3 From bf0c93c21103a02f9efe4c427b9fefd6c5732666 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 10 Jan 2011 21:19:11 +0100 Subject: Use title index for more than 11 p_first_heading calls This change makes p_get_first_heading load the title index when more than 11 requests that caused a call to p_get_metadata have already been done. This means that small pages and the breadcrums won't trigger the loading of the title index but for larger pages or the sitemap the title index will be used. This is necessary because every call to p_get_metadata can trigger the parsing and rendering of a whole page and there can be many calls when useheading is activated and e.g. the index/sitemap page is displayed. Additionally this adds a small title cache that caches titles requested from p_get_metadata. Further tests should be done how this affects memory usage and how often the index loading is triggered in order to see if that parameter should be adjusted. --- inc/parserutils.php | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index 9224dae10..b7359d7ef 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -625,9 +625,41 @@ function & p_get_renderer($mode) { * headings ... and so on. * * @author Andreas Gohr + * @author Michael Hamann */ function p_get_first_heading($id, $render=true){ - return p_get_metadata($id,'title',$render); + // counter how many titles have been requested using p_get_metadata + static $count = 0; + // the index of all titles, only loaded when many titles are requested + static $title_index = null; + // cache for titles requested using p_get_metadata + static $title_cache = array(); + + $id = cleanID($id); + + // check if this title has already been requested + if (isset($title_cache[$id])) + return $title_cache[$id]; + + // check if already too many titles have been requested and probably + // using the title index is better + if ($count > 10) { + if (is_null($title_index)) { + $pages = array_map('rtrim', idx_getIndex('page', '')); + $titles = array_map('rtrim', idx_getIndex('title', '')); + // check for corrupt title index #FS2076 + if(count($pages) != count($titles)){ + $titles = array_fill(0,count($pages),''); + @unlink($conf['indexdir'].'/title.idx'); // will be rebuilt in inc/init.php + } + $title_index = array_combine($pages, $titles); + } + return $title_index[$id]; + } + + ++$count; + $title_cache[$id] = p_get_metadata($id,'title',$render); + return $title_cache[$id]; } /** -- cgit v1.2.3 From ff725173bf45c47b1ed9778524710cce72b1d42d Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Mon, 10 Jan 2011 23:41:43 +0100 Subject: Add define for metadata usage limit in p_get_first_heading This commit introduces a new define P_GET_FIRST_HEADING_METADATA_LIMIT that can be set in preload.php in order to change the limit for how many pages the first heading shall be loaded from metadata in p_get_first_heading. Changing this is probably most interesting for Wikis with a lot of pages where loading the title index costs a significant amount of time and memory. --- inc/parserutils.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index b7359d7ef..6e349e984 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -9,6 +9,13 @@ if(!defined('DOKU_INC')) die('meh.'); +/** + * For how many different pages shall the first heading be loaded from the + * metadata? When this limit is reached the title index is loaded and used for + * all following requests. + */ +if (!defined('P_GET_FIRST_HEADING_METADATA_LIMIT')) define('P_GET_FIRST_HEADING_METADATA_LIMIT', 10); + /** * Returns the parsed Wikitext in XHTML for the given id and revision. * @@ -629,7 +636,7 @@ function & p_get_renderer($mode) { */ function p_get_first_heading($id, $render=true){ // counter how many titles have been requested using p_get_metadata - static $count = 0; + static $count = 1; // the index of all titles, only loaded when many titles are requested static $title_index = null; // cache for titles requested using p_get_metadata @@ -643,7 +650,7 @@ function p_get_first_heading($id, $render=true){ // check if already too many titles have been requested and probably // using the title index is better - if ($count > 10) { + if ($count > P_GET_FIRST_HEADING_METADATA_LIMIT) { if (is_null($title_index)) { $pages = array_map('rtrim', idx_getIndex('page', '')); $titles = array_map('rtrim', idx_getIndex('title', '')); -- cgit v1.2.3 From 0e5fde485b65b5a64fef50214496ad24bed17cef Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Sun, 6 Feb 2011 13:03:00 +0100 Subject: Allow p_set_metadata during rendering, test cases included. FS#1827 --- inc/parserutils.php | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index 6e349e984..86297da8b 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -291,18 +291,25 @@ function p_get_metadata($id, $key='', $render=true){ * @return boolean true on success * * @author Esther Brunner + * @author Michael Hamann */ function p_set_metadata($id, $data, $render=false, $persistent=true){ if (!is_array($data)) return false; - global $ID; + global $ID, $METADATA_RENDERERS; - // cache the current page - $cache = ($ID == $id); - $orig = p_read_metadata($id, $cache); + // if there is currently a renderer change the data in the renderer instead + if (isset($METADATA_RENDERERS[$id])) { + $orig =& $METADATA_RENDERERS[$id]; + $meta = $orig; + } else { + // cache the current page + $cache = ($ID == $id); + $orig = p_read_metadata($id, $cache); - // render metadata first? - $meta = $render ? p_render_metadata($id, $orig) : $orig; + // render metadata first? + $meta = $render ? p_render_metadata($id, $orig) : $orig; + } // now add the passed metadata $protected = array('description', 'date', 'contributor'); @@ -339,7 +346,13 @@ function p_set_metadata($id, $data, $render=false, $persistent=true){ // save only if metadata changed if ($meta == $orig) return true; - return p_save_metadata($id, $meta); + if (isset($METADATA_RENDERERS[$id])) { + // set both keys individually as the renderer has references to the individual keys + $METADATA_RENDERERS[$id]['current'] = $meta['current']; + $METADATA_RENDERERS[$id]['persistent'] = $meta['persistent']; + } else { + return p_save_metadata($id, $meta); + } } /** @@ -413,7 +426,15 @@ function p_save_metadata($id, $meta) { */ function p_render_metadata($id, $orig){ // make sure the correct ID is in global ID - global $ID; + global $ID, $METADATA_RENDERERS; + + // avoid recursive rendering processes for the same id + if (isset($METADATA_RENDERERS[$id])) + return $orig; + + // store the original metadata in the global $METADATA_RENDERERS so p_set_metadata can use it + $METADATA_RENDERERS[$id] =& $orig; + $keep = $ID; $ID = $id; @@ -428,13 +449,14 @@ function p_render_metadata($id, $orig){ $instructions = p_cached_instructions(wikiFN($id),false,$id); if(is_null($instructions)){ $ID = $keep; + unset($METADATA_RENDERERS[$id]); return null; // something went wrong with the instructions } // set up the renderer $renderer = new Doku_Renderer_metadata(); - $renderer->meta = $orig['current']; - $renderer->persistent = $orig['persistent']; + $renderer->meta =& $orig['current']; + $renderer->persistent =& $orig['persistent']; // loop through the instructions foreach ($instructions as $instruction){ @@ -442,11 +464,13 @@ function p_render_metadata($id, $orig){ call_user_func_array(array(&$renderer, $instruction[0]), (array) $instruction[1]); } - $evt->result = array('current'=>$renderer->meta,'persistent'=>$renderer->persistent); + $evt->result = array('current'=>&$renderer->meta,'persistent'=>&$renderer->persistent); } $evt->advise_after(); + // clean up $ID = $keep; + unset($METADATA_RENDERERS[$id]); return $evt->result; } -- cgit v1.2.3 From 79c1bbfeeff1769b09df93db588db6cfbbaf6971 Mon Sep 17 00:00:00 2001 From: Michael Hamann Date: Sun, 6 Feb 2011 13:03:57 +0100 Subject: p_get_metadata: show the save error message only when metadata has been changed --- inc/parserutils.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'inc/parserutils.php') diff --git a/inc/parserutils.php b/inc/parserutils.php index 86297da8b..9b2d99328 100644 --- a/inc/parserutils.php +++ b/inc/parserutils.php @@ -258,7 +258,7 @@ function p_get_metadata($id, $key='', $render=true){ if ($meta == $old_meta || p_save_metadata($id, $meta)) { // store a timestamp in order to make sure that the cachefile is touched $cachefile->storeCache(time()); - } else { + } elseif ($meta != $old_meta) { msg('Unable to save metadata file. Hint: disk full; file permissions; safe_mode setting.',-1); } } -- cgit v1.2.3