From 5b037208e131484949e86b50b6dd198dfd117708 Mon Sep 17 00:00:00 2001
From: Gina Haeussge <osd@foosel.net>
Date: Sun, 22 Jun 2008 11:21:05 +0200
Subject: HTTPClient: Allow limited retrieval without aborting

Introduces a new member variable "max_bodysize_abort" into the HTTP client.
If this is set to true (which is the default), the HTTP client shows it
current behvaiour of aborting with an error condition if the given max_bodysize
is exceeded during retrieval. If it is set to false however, the HTTP client
just stops retrieval but returns what it already got. This allows the retrieval
of e.g. the first 10K of a web document for searching for pingback data.

darcs-hash:20080622092105-2b4f5-28c4399123775d82986faf22c7d89ccb22a8e025.gz
---
 inc/HTTPClient.php | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'inc/HTTPClient.php')

diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php
index ed0a3aec6..7474e1fd8 100644
--- a/inc/HTTPClient.php
+++ b/inc/HTTPClient.php
@@ -58,7 +58,8 @@ class HTTPClient {
     var $cookies;
     var $referer;
     var $max_redirect;
-    var $max_bodysize;  // abort if the response body is bigger than this
+    var $max_bodysize;
+    var $max_bodysize_abort = true;  // if set, abort if the response body is bigger than max_bodysize
     var $header_regexp; // if set this RE must match against the headers, else abort
     var $headers;
     var $debug;
@@ -252,7 +253,8 @@ class HTTPClient {
         if($this->max_bodysize && preg_match('/\r?\nContent-Length:\s*(\d+)\r?\n/i',$r_headers,$match)){
             if($match[1] > $this->max_bodysize){
                 $this->error = 'Reported content length exceeds allowed response size';
-                return false;
+                if ($this->max_bodysize_abort)
+                    return false;
             }
         }
 
@@ -327,7 +329,10 @@ class HTTPClient {
 
                 if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){
                     $this->error = 'Allowed response size exceeded';
-                    return false;
+                    if ($this->max_bodysize_abort)
+                        return false;
+                    else
+                        break;
                 }
             } while ($chunk_size);
         }else{
@@ -342,7 +347,10 @@ class HTTPClient {
                 $r_size = strlen($r_body);
                 if($this->max_bodysize && $r_size > $this->max_bodysize){
                     $this->error = 'Allowed response size exceeded';
-                    return false;
+                    if ($this->max_bodysize_abort)
+                        return false;
+                    else
+                        break;
                 }
                 if($this->resp_headers['content-length'] && !$this->resp_headers['transfer-encoding'] &&
                    $this->resp_headers['content-length'] == $r_size){
-- 
cgit v1.2.3