From 369075828e13e37a65a2f8062a74e89f98dd3fac Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Wed, 6 May 2015 18:21:17 -0400 Subject: Append to BZip2 files. Unit tests for writing files. --- inc/io.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 0636a4b62..8846b7e56 100644 --- a/inc/io.php +++ b/inc/io.php @@ -223,7 +223,16 @@ function io_saveFile($file,$content,$append=false){ gzwrite($fh, $content); gzclose($fh); }else if(substr($file,-4) == '.bz2'){ - $fh = @bzopen($file,$mode{0}); + if($append) { + $bzcontent = bzfile($file); + if($bzcontent === false) { + msg("Writing $file failed", -1); + io_unlock($file); + return false; + } + $content = $bzcontent.$content; + } + $fh = @bzopen($file,'w'); if(!$fh){ msg("Writing $file failed", -1); io_unlock($file); -- cgit v1.2.3 From cfb71e37ca8859c4ad9a1db73de0a293ffc7a902 Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Wed, 6 May 2015 23:31:39 -0400 Subject: Deleting lines works with BZ2 files. --- inc/io.php | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 8846b7e56..b8a77b730 100644 --- a/inc/io.php +++ b/inc/io.php @@ -127,22 +127,36 @@ function io_readFile($file,$clean=true){ * @author Andreas Gohr * * @param string $file filename - * @return string|bool content or false on error + * @param bool $array return array of lines + * @return string|array|bool content or false on error */ -function bzfile($file){ +function bzfile($file, $array=false) { $bz = bzopen($file,"r"); if($bz === false) return false; + if($array) $lines = array(); $str = ''; - while (!feof($bz)){ + while (!feof($bz)) { //8192 seems to be the maximum buffersize? $buffer = bzread($bz,8192); if(($buffer === false) || (bzerrno($bz) !== 0)) { return false; } $str = $str . $buffer; + if($array) { + $pos = strpos($str, "\n"); + while($pos !== false) { + $lines[] = substr($str, 0, $pos+1); + $str = substr($str, $pos+1); + $pos = strpos($str, "\n"); + } + } } bzclose($bz); + if($array) { + if($str !== '') $lines[] = $str; + return $lines; + } return $str; } @@ -280,6 +294,8 @@ function io_deleteFromFile($file,$badline,$regex=false){ // load into array if(substr($file,-3) == '.gz'){ $lines = gzfile($file); + }else if(substr($file,-4) == '.bz2'){ + $lines = bzfile($file, true); }else{ $lines = file($file); } @@ -306,6 +322,15 @@ function io_deleteFromFile($file,$badline,$regex=false){ } gzwrite($fh, $content); gzclose($fh); + }else if(substr($file,-4) == '.bz2'){ + $fh = @bzopen($file,'w'); + if(!$fh){ + msg("Removing content from $file failed",-1); + io_unlock($file); + return false; + } + bzwrite($fh, $content); + bzclose($fh); }else{ $fh = @fopen($file,'wb'); if(!$fh){ -- cgit v1.2.3 From 1bd6bbdebc26f9cd916f8f287cd2cabc07bee8d1 Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Thu, 7 May 2015 01:37:11 -0400 Subject: Add io_replaceInFile --- inc/io.php | 158 +++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 86 insertions(+), 72 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index b8a77b730..11a3fa77f 100644 --- a/inc/io.php +++ b/inc/io.php @@ -205,13 +205,7 @@ function _io_writeWikiPage_action($data) { } /** - * Saves $content to $file. - * - * If the third parameter is set to true the given content - * will be appended. - * - * Uses gzip if extension is .gz - * and bz2 if extension is .bz2 + * Internal function to save contents to a file. * * @author Andreas Gohr * @@ -220,73 +214,90 @@ function _io_writeWikiPage_action($data) { * @param bool $append * @return bool true on success, otherwise false */ -function io_saveFile($file,$content,$append=false){ +function _io_saveFile($file, $content, $append) { global $conf; $mode = ($append) ? 'ab' : 'wb'; - $fileexists = file_exists($file); - io_makeFileDir($file); - io_lock($file); + if(substr($file,-3) == '.gz'){ $fh = @gzopen($file,$mode.'9'); - if(!$fh){ - msg("Writing $file failed",-1); - io_unlock($file); - return false; - } + if(!$fh) return false; gzwrite($fh, $content); gzclose($fh); }else if(substr($file,-4) == '.bz2'){ if($append) { $bzcontent = bzfile($file); - if($bzcontent === false) { - msg("Writing $file failed", -1); - io_unlock($file); - return false; - } + if($bzcontent === false) return false; $content = $bzcontent.$content; } $fh = @bzopen($file,'w'); - if(!$fh){ - msg("Writing $file failed", -1); - io_unlock($file); - return false; - } + if(!$fh) return false; bzwrite($fh, $content); bzclose($fh); }else{ $fh = @fopen($file,$mode); - if(!$fh){ - msg("Writing $file failed",-1); - io_unlock($file); - return false; - } + if(!$fh) return false; fwrite($fh, $content); fclose($fh); } if(!$fileexists and !empty($conf['fperm'])) chmod($file, $conf['fperm']); - io_unlock($file); return true; } /** - * Delete exact linematch for $badline from $file. + * Saves $content to $file. * - * Be sure to include the trailing newline in $badline + * If the third parameter is set to true the given content + * will be appended. * * Uses gzip if extension is .gz + * and bz2 if extension is .bz2 * - * 2005-10-14 : added regex option -- Christopher Smith + * @author Andreas Gohr * - * @author Steven Danz + * @param string $file filename path to file + * @param string $content + * @param bool $append + * @return bool true on success, otherwise false + */ +function io_saveFile($file, $content, $append=false) { + io_makeFileDir($file); + io_lock($file); + if(!_io_saveFile($file, $content, $append)) { + msg("Writing $file failed",-1); + io_unlock($file); + return false; + } + io_unlock($file); + return true; +} + +/** + * Replace one or more occurrences of a line in a file. * - * @param string $file filename - * @param string $badline exact linematch to remove - * @param bool $regex use regexp? + * The default, when $maxlines is 0 is to delete all matches then append a single line. + * If $maxlines is -1, then every $oldline will be replaced with $newline, and $regex is true + * then preg captures are used. If $maxlines is greater than 0 then the first $maxlines + * matches are replaced with $newline. + * + * Be sure to include the trailing newline in $oldline + * + * Uses gzip if extension is .gz + * and bz2 if extension is .bz2 + * + * @author Steven Danz + * @author Christopher Smith + * @author Patrick Brown + * + * @param string $file filename + * @param string $oldline exact linematch to remove + * @param string $newline new line to insert + * @param bool $regex use regexp? + * @param int $maxlines number of occurrences of the line to replace * @return bool true on success */ -function io_deleteFromFile($file,$badline,$regex=false){ +function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) { if (!file_exists($file)) return true; io_lock($file); @@ -302,44 +313,31 @@ function io_deleteFromFile($file,$badline,$regex=false){ // remove all matching lines if ($regex) { - $lines = preg_grep($badline,$lines,PREG_GREP_INVERT); + if($maxlines == 0) { + $lines = preg_grep($oldline, $lines, PREG_GREP_INVERT); + } else { + $lines = preg_replace($oldline, $newline, $lines, $maxlines); + } } else { - $pos = array_search($badline,$lines); //return null or false if not found + $count = 0; + $replaceline = $maxlines == 0 ? '' : $newline; + $pos = array_search($oldline,$lines); //return null or false if not found while(is_int($pos)){ - unset($lines[$pos]); - $pos = array_search($badline,$lines); + $lines[$pos] = $replaceline; + if($maxlines > 0 && ++$count >= $maxlines) break; + $pos = array_search($oldline,$lines); } } + if($maxlines == 0 && ((string)$newline) !== '') { + $lines[] = $newline; + } + if(count($lines)){ - $content = join('',$lines); - if(substr($file,-3) == '.gz'){ - $fh = @gzopen($file,'wb9'); - if(!$fh){ - msg("Removing content from $file failed",-1); - io_unlock($file); - return false; - } - gzwrite($fh, $content); - gzclose($fh); - }else if(substr($file,-4) == '.bz2'){ - $fh = @bzopen($file,'w'); - if(!$fh){ - msg("Removing content from $file failed",-1); - io_unlock($file); - return false; - } - bzwrite($fh, $content); - bzclose($fh); - }else{ - $fh = @fopen($file,'wb'); - if(!$fh){ - msg("Removing content from $file failed",-1); - io_unlock($file); - return false; - } - fwrite($fh, $content); - fclose($fh); + if(!_io_saveFile($file, join('',$lines), false)) { + msg("Removing content from $file failed",-1); + io_unlock($file); + return false; } }else{ @unlink($file); @@ -349,6 +347,22 @@ function io_deleteFromFile($file,$badline,$regex=false){ return true; } +/** + * Delete lines that match $badline from $file. + * + * Be sure to include the trailing newline in $badline + * + * @author Patrick Brown + * + * @param string $file filename + * @param string $badline exact linematch to remove + * @param bool $regex use regexp? + * @return bool true on success + */ +function io_deleteFromFile($file,$badline,$regex=false){ + return io_replaceInFile($file,$badline,null,$regex,0); +} + /** * Tries to lock a file * -- cgit v1.2.3 From 6c0002048504e43b399abece0668afa2b5c87a07 Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Fri, 8 May 2015 17:11:44 -0400 Subject: Limit number of lines to replace. --- inc/io.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 11a3fa77f..dbb42114b 100644 --- a/inc/io.php +++ b/inc/io.php @@ -313,10 +313,16 @@ function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) // remove all matching lines if ($regex) { - if($maxlines == 0) { - $lines = preg_grep($oldline, $lines, PREG_GREP_INVERT); + if($maxlines > 0) { + $matches = preg_grep($oldline, $lines); + $count = 0; + foreach($matches as $ix=>$m) { + $lines[$ix] = preg_replace($oldline, $newline, $m); + if(++$count >= $maxlines) break; + } } else { - $lines = preg_replace($oldline, $newline, $lines, $maxlines); + $lines = ($maxlines == 0) ? preg_grep($oldline, $lines, PREG_GREP_INVERT) + : preg_replace($oldline, $newline, $lines, $maxlines); } } else { $count = 0; -- cgit v1.2.3 From 9a734b7aaba1445e06c1ccb95e59f54e01688d45 Mon Sep 17 00:00:00 2001 From: Christopher Smith Date: Thu, 28 May 2015 16:37:32 +0100 Subject: Refactor code to make it simpler. The changes should also: - fix unlikely edge case when replacement line is the same as the old line (would have resulted in timeout) - reduce memory footprint - avoid applying string search beyond maxlines replacement limit --- inc/io.php | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index dbb42114b..51ca2ea14 100644 --- a/inc/io.php +++ b/inc/io.php @@ -311,28 +311,23 @@ function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) $lines = file($file); } - // remove all matching lines - if ($regex) { - if($maxlines > 0) { - $matches = preg_grep($oldline, $lines); - $count = 0; - foreach($matches as $ix=>$m) { - $lines[$ix] = preg_replace($oldline, $newline, $m); - if(++$count >= $maxlines) break; - } - } else { - $lines = ($maxlines == 0) ? preg_grep($oldline, $lines, PREG_GREP_INVERT) - : preg_replace($oldline, $newline, $lines, $maxlines); - } - } else { + // make non-regexes into regexes + $pattern = $regex ? $oldline : '/'.preg_quote($oldline,'/').'/'; + $replace = $regex ? $newline : addcslashes($newline, '\$'); + + // remove matching lines + if ($maxlines > 0) { $count = 0; - $replaceline = $maxlines == 0 ? '' : $newline; - $pos = array_search($oldline,$lines); //return null or false if not found - while(is_int($pos)){ - $lines[$pos] = $replaceline; - if($maxlines > 0 && ++$count >= $maxlines) break; - $pos = array_search($oldline,$lines); + $matched = 0; + while (($count < $maxlines) && (list($i,$line) = each($lines))) { + // $matched will be set to 0|1 depending on whether pattern is matched and line replaced + $lines[$i] = preg_replace($pattern, $replace, $line, -1, $matched); + if ($matched) $count++; } + } else { + $lines = ($maxlines == 0) ? + preg_grep($pattern, $lines, PREG_GREP_INVERT) : + preg_replace($pattern, $replace, $lines); } if($maxlines == 0 && ((string)$newline) !== '') { -- cgit v1.2.3 From 3dfe7d64760baa568018c1d6d311c26d1a2da098 Mon Sep 17 00:00:00 2001 From: Christopher Smith Date: Fri, 29 May 2015 16:52:05 +0100 Subject: add anchors when constructing pattern from a non-regex oldline --- inc/io.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 51ca2ea14..c559feb17 100644 --- a/inc/io.php +++ b/inc/io.php @@ -312,7 +312,7 @@ function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) } // make non-regexes into regexes - $pattern = $regex ? $oldline : '/'.preg_quote($oldline,'/').'/'; + $pattern = $regex ? $oldline : '/^'.preg_quote($oldline,'/').'$/'; $replace = $regex ? $newline : addcslashes($newline, '\$'); // remove matching lines -- cgit v1.2.3 From e12c5ac781d560502d478775502df70cd80472de Mon Sep 17 00:00:00 2001 From: Christopher Smith Date: Fri, 29 May 2015 16:55:23 +0100 Subject: Minor Refactoring - put test comments in more appropriate spot - move appending replacement line alongside its search/delete code --- inc/io.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index c559feb17..4c7fb094f 100644 --- a/inc/io.php +++ b/inc/io.php @@ -324,14 +324,14 @@ function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) $lines[$i] = preg_replace($pattern, $replace, $line, -1, $matched); if ($matched) $count++; } - } else { - $lines = ($maxlines == 0) ? - preg_grep($pattern, $lines, PREG_GREP_INVERT) : - preg_replace($pattern, $replace, $lines); - } + } else if ($maxlines == 0) { + $lines = preg_grep($pattern, $lines, PREG_GREP_INVERT); - if($maxlines == 0 && ((string)$newline) !== '') { - $lines[] = $newline; + if ((string)$newline !== ''){ + $lines[] = $newline; + } + } else { + $lines = preg_replace($pattern, $replace, $lines); } if(count($lines)){ -- cgit v1.2.3 From dc4a4eb00d67d7d28fae137437900220920577d4 Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Fri, 29 May 2015 15:38:43 -0400 Subject: Abort io_replaceInLine when the search parameter is empty --- inc/io.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 4c7fb094f..6d3c20047 100644 --- a/inc/io.php +++ b/inc/io.php @@ -298,6 +298,11 @@ function io_saveFile($file, $content, $append=false) { * @return bool true on success */ function io_replaceInFile($file, $oldline, $newline, $regex=false, $maxlines=0) { + if ((string)$oldline === '') { + trigger_error('$oldline parameter cannot be empty in io_replaceInFile()', E_USER_WARNING); + return false; + } + if (!file_exists($file)) return true; io_lock($file); -- cgit v1.2.3 From d93ba631117932f06b44535a9d6256cc8e9c4b90 Mon Sep 17 00:00:00 2001 From: Patrick Brown Date: Fri, 29 May 2015 15:46:19 -0400 Subject: Rephrase description of io_replaceInFiles to be clarify use of regex --- inc/io.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 6d3c20047..9be648824 100644 --- a/inc/io.php +++ b/inc/io.php @@ -276,12 +276,14 @@ function io_saveFile($file, $content, $append=false) { /** * Replace one or more occurrences of a line in a file. * - * The default, when $maxlines is 0 is to delete all matches then append a single line. - * If $maxlines is -1, then every $oldline will be replaced with $newline, and $regex is true - * then preg captures are used. If $maxlines is greater than 0 then the first $maxlines - * matches are replaced with $newline. + * The default, when $maxlines is 0 is to delete all matching lines then append a single line. + * A regex that matches any part of the line will remove the entire line in this mode. + * Captures in $newline are not available. * - * Be sure to include the trailing newline in $oldline + * Otherwise each line is matched and replaced individually, up to the first $maxlines lines + * or all lines if $maxlines is -1. If $regex is true then captures can be used in $newline. + * + * Be sure to include the trailing newline in $oldline when replacing entire lines. * * Uses gzip if extension is .gz * and bz2 if extension is .bz2 -- cgit v1.2.3 From 2ad45addfac44b12c095303cf7abe488576b0802 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Fri, 24 Jul 2015 15:18:19 +0200 Subject: avoid errors on trying to read corrupt gzip files --- inc/io.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'inc/io.php') diff --git a/inc/io.php b/inc/io.php index 9be648824..704c5b1a6 100644 --- a/inc/io.php +++ b/inc/io.php @@ -107,13 +107,15 @@ function io_readFile($file,$clean=true){ $ret = ''; if(file_exists($file)){ if(substr($file,-3) == '.gz'){ - $ret = join('',gzfile($file)); + $ret = gzfile($file); + if(is_array($ret)) $ret = join('', $ret); }else if(substr($file,-4) == '.bz2'){ $ret = bzfile($file); }else{ $ret = file_get_contents($file); } } + if($ret === null) return false; if($ret !== false && $clean){ return cleanText($ret); }else{ -- cgit v1.2.3