From 020ab112b3d5b5c640eb1a297ec77ff73cf2b0d7 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sat, 16 Jun 2012 02:30:00 -0400 Subject: Reference static variable through 'self::' as you're supposed to. --- inc/HTTPClient.php | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 26bee52a7..bb87056e2 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -284,11 +284,11 @@ class HTTPClient { // already connected? $connectionId = $this->_uniqueConnectionId($server,$port); - $this->_debug('connection pool', $this->connections); + $this->_debug('connection pool', self::$connections); $socket = null; - if (isset($this->connections[$connectionId])) { + if (isset(self::$connections[$connectionId])) { $this->_debug('reusing connection', $connectionId); - $socket = $this->connections[$connectionId]; + $socket = self::$connections[$connectionId]; } if (is_null($socket) || feof($socket)) { $this->_debug('opening connection', $connectionId); @@ -302,9 +302,9 @@ class HTTPClient { // keep alive? if ($this->keep_alive) { - $this->connections[$connectionId] = $socket; + self::$connections[$connectionId] = $socket; } else { - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); } } @@ -333,7 +333,7 @@ class HTTPClient { if(time()-$start > $this->timeout){ $this->status = -100; $this->error = sprintf('Timeout while sending request (%.3fs)',$this->_time() - $this->start); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } @@ -348,7 +348,7 @@ class HTTPClient { if($ret === false){ $this->status = -100; $this->error = 'Failed writing to socket'; - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } $written += $ret; @@ -363,12 +363,12 @@ class HTTPClient { if(time()-$start > $this->timeout){ $this->status = -100; $this->error = sprintf('Timeout while reading headers (%.3fs)',$this->_time() - $this->start); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } if(feof($socket)){ $this->error = 'Premature End of File (socket)'; - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } usleep(1000); @@ -382,7 +382,7 @@ class HTTPClient { if($match[1] > $this->max_bodysize){ $this->error = 'Reported content length exceeds allowed response size'; if ($this->max_bodysize_abort) - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } } @@ -390,7 +390,7 @@ class HTTPClient { // get Status if (!preg_match('/^HTTP\/(\d\.\d)\s*(\d+).*?\n/', $r_headers, $m)) { $this->error = 'Server returned bad answer'; - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } $this->status = $m[2]; @@ -419,7 +419,7 @@ class HTTPClient { // close the connection because we don't handle content retrieval here // that's the easiest way to clean up the connection fclose($socket); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); if (empty($this->resp_headers['location'])){ $this->error = 'Redirect but no Location Header found'; @@ -448,7 +448,7 @@ class HTTPClient { // check if headers are as expected if($this->header_regexp && !preg_match($this->header_regexp,$r_headers)){ $this->error = 'The received headers did not match the given regexp'; - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } @@ -460,13 +460,13 @@ class HTTPClient { do { if(feof($socket)){ $this->error = 'Premature End of File (socket)'; - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } if(time()-$start > $this->timeout){ $this->status = -100; $this->error = sprintf('Timeout while reading chunk (%.3fs)',$this->_time() - $this->start); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } $byte = fread($socket,1); @@ -484,7 +484,7 @@ class HTTPClient { if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){ $this->error = 'Allowed response size exceeded'; if ($this->max_bodysize_abort){ - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } else { break; @@ -497,7 +497,7 @@ class HTTPClient { if(time()-$start > $this->timeout){ $this->status = -100; $this->error = sprintf('Timeout while reading response (%.3fs)',$this->_time() - $this->start); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } $r_body .= fread($socket,4096); @@ -505,7 +505,7 @@ class HTTPClient { if($this->max_bodysize && $r_size > $this->max_bodysize){ $this->error = 'Allowed response size exceeded'; if ($this->max_bodysize_abort) { - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); return false; } else { break; @@ -525,7 +525,7 @@ class HTTPClient { // close socket $status = socket_get_status($socket); fclose($socket); - unset($this->connections[$connectionId]); + unset(self::$connections[$connectionId]); } // decode gzip if needed -- cgit v1.2.3 From a09831ea8abf0abcf3e7f72f0974dc5e0fc301c8 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sat, 16 Jun 2012 02:34:47 -0400 Subject: Avoid strict warnings about unset array keys. --- inc/HTTPClient.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index bb87056e2..524dd9a2c 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -227,7 +227,7 @@ class HTTPClient { $path = $uri['path']; if(empty($path)) $path = '/'; if(!empty($uri['query'])) $path .= '?'.$uri['query']; - if(isset($uri['port']) && !empty($uri['port'])) $port = $uri['port']; + if(!empty($uri['port'])) $port = $uri['port']; if(isset($uri['user'])) $this->user = $uri['user']; if(isset($uri['pass'])) $this->pass = $uri['pass']; @@ -249,7 +249,7 @@ class HTTPClient { // prepare headers $headers = $this->headers; $headers['Host'] = $uri['host']; - if($uri['port']) $headers['Host'].= ':'.$uri['port']; + if(!empty($uri['port'])) $headers['Host'].= ':'.$uri['port']; $headers['User-Agent'] = $this->agent; $headers['Referer'] = $this->referer; if ($this->keep_alive) { @@ -583,7 +583,7 @@ class HTTPClient { $lines = explode("\n",$string); array_shift($lines); //skip first line (status) foreach($lines as $line){ - list($key, $val) = explode(':',$line,2); + @list($key, $val) = explode(':',$line,2); $key = trim($key); $val = trim($val); $key = strtolower($key); -- cgit v1.2.3 From fc6c2b2f78f7278af30dd2dd0758e90ae771322e Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sat, 16 Jun 2012 03:23:17 -0400 Subject: Correct handling of chunked transfer encoding. (FS#2535) --- inc/HTTPClient.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 524dd9a2c..a07d263dd 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -456,7 +456,7 @@ class HTTPClient { $r_body = ''; if(preg_match('/transfer\-(en)?coding:\s*chunked\r\n/i',$r_headers)){ do { - unset($chunk_size); + $chunk_size = ''; do { if(feof($socket)){ $this->error = 'Premature End of File (socket)'; @@ -471,13 +471,17 @@ class HTTPClient { } $byte = fread($socket,1); $chunk_size .= $byte; - } while (preg_match('/[a-zA-Z0-9]/',$byte)); // read chunksize including \r + } while (preg_match('/^[a-zA-Z0-9]?$/',$byte)); // read chunksize including \r $byte = fread($socket,1); // readtrailing \n $chunk_size = hexdec($chunk_size); if ($chunk_size) { - $this_chunk = fread($socket,$chunk_size); - $r_body .= $this_chunk; + $read_size = $chunk_size; + while ($read_size > 0) { + $this_chunk = fread($socket,$read_size); + $r_body .= $this_chunk; + $read_size -= strlen($this_chunk); + } $byte = fread($socket,2); // read trailing \r\n } -- cgit v1.2.3 From 189ba183ccac1cf7a2d02756895207c48431f70b Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sat, 16 Jun 2012 23:02:26 -0400 Subject: Raise an exception on socket errors. This is the first step in refactoring the socket reader to be more resilient and easier to debug. --- inc/HTTPClient.php | 352 +++++++++++++++++++++++++---------------------------- 1 file changed, 167 insertions(+), 185 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index a07d263dd..f11621f6a 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -61,6 +61,8 @@ class DokuHTTPClient extends HTTPClient { } +class HTTPClientException extends Exception { } + /** * This class implements a basic HTTP client * @@ -308,220 +310,200 @@ class HTTPClient { } } - //set blocking - stream_set_blocking($socket,1); - - // build request - $request = "$method $request_url HTTP/".$this->http.HTTP_NL; - $request .= $this->_buildHeaders($headers); - $request .= $this->_getCookies(); - $request .= HTTP_NL; - $request .= $data; - - $this->_debug('request',$request); - - // select parameters - $sel_r = null; - $sel_w = array($socket); - $sel_e = null; - - // send request - $towrite = strlen($request); - $written = 0; - while($written < $towrite){ - // check timeout - if(time()-$start > $this->timeout){ - $this->status = -100; - $this->error = sprintf('Timeout while sending request (%.3fs)',$this->_time() - $this->start); - unset(self::$connections[$connectionId]); - return false; - } - - // wait for stream ready or timeout (1sec) - if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ - usleep(1000); - continue; - } - - // write to stream - $ret = fwrite($socket, substr($request,$written,4096)); - if($ret === false){ - $this->status = -100; - $this->error = 'Failed writing to socket'; - unset(self::$connections[$connectionId]); - return false; - } - $written += $ret; - } - - // continue non-blocking - stream_set_blocking($socket,0); + try { + //set blocking + stream_set_blocking($socket,1); + + // build request + $request = "$method $request_url HTTP/".$this->http.HTTP_NL; + $request .= $this->_buildHeaders($headers); + $request .= $this->_getCookies(); + $request .= HTTP_NL; + $request .= $data; + + $this->_debug('request',$request); + + // select parameters + $sel_r = null; + $sel_w = array($socket); + $sel_e = null; + + // send request + $towrite = strlen($request); + $written = 0; + while($written < $towrite){ + // check timeout + if(time()-$start > $this->timeout) + throw new HTTPClientException(sprintf('Timeout while sending request (%.3fs)',$this->_time() - $this->start), -100); + + // wait for stream ready or timeout (1sec) + if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ + usleep(1000); + continue; + } - // read headers from socket - $r_headers = ''; - do{ - if(time()-$start > $this->timeout){ - $this->status = -100; - $this->error = sprintf('Timeout while reading headers (%.3fs)',$this->_time() - $this->start); - unset(self::$connections[$connectionId]); - return false; - } - if(feof($socket)){ - $this->error = 'Premature End of File (socket)'; - unset(self::$connections[$connectionId]); - return false; + // write to stream + $ret = fwrite($socket, substr($request,$written,4096)); + if($ret === false) + throw new HTTPClientException('Failed writing to socket', -100); + $written += $ret; } - usleep(1000); - $r_headers .= fgets($socket,1024); - }while(!preg_match('/\r?\n\r?\n$/',$r_headers)); - $this->_debug('response headers',$r_headers); + // continue non-blocking + stream_set_blocking($socket,0); - // check if expected body size exceeds allowance - if($this->max_bodysize && preg_match('/\r?\nContent-Length:\s*(\d+)\r?\n/i',$r_headers,$match)){ - if($match[1] > $this->max_bodysize){ - $this->error = 'Reported content length exceeds allowed response size'; - if ($this->max_bodysize_abort) - unset(self::$connections[$connectionId]); - return false; + // read headers from socket + $r_headers = ''; + do{ + if(time()-$start > $this->timeout) + throw new HTTPClientException(sprintf('Timeout while reading headers (%.3fs)',$this->_time() - $this->start), -100); + if(feof($socket)) + throw new HTTPClientException('Premature End of File (socket)'); + usleep(1000); + $r_headers .= fgets($socket,1024); + }while(!preg_match('/\r?\n\r?\n$/',$r_headers)); + + $this->_debug('response headers',$r_headers); + + // check if expected body size exceeds allowance + if($this->max_bodysize && preg_match('/\r?\nContent-Length:\s*(\d+)\r?\n/i',$r_headers,$match)){ + if($match[1] > $this->max_bodysize){ + if ($this->max_bodysize_abort) + throw new HTTPClientException('Reported content length exceeds allowed response size'); + else + $this->error = 'Reported content length exceeds allowed response size'; + } } - } - // get Status - if (!preg_match('/^HTTP\/(\d\.\d)\s*(\d+).*?\n/', $r_headers, $m)) { - $this->error = 'Server returned bad answer'; - unset(self::$connections[$connectionId]); - return false; - } - $this->status = $m[2]; - - // handle headers and cookies - $this->resp_headers = $this->_parseHeaders($r_headers); - if(isset($this->resp_headers['set-cookie'])){ - foreach ((array) $this->resp_headers['set-cookie'] as $cookie){ - list($cookie) = explode(';',$cookie,2); - list($key,$val) = explode('=',$cookie,2); - $key = trim($key); - if($val == 'deleted'){ - if(isset($this->cookies[$key])){ - unset($this->cookies[$key]); + // get Status + if (!preg_match('/^HTTP\/(\d\.\d)\s*(\d+).*?\n/', $r_headers, $m)) + throw new HTTPClientException('Server returned bad answer'); + + $this->status = $m[2]; + + // handle headers and cookies + $this->resp_headers = $this->_parseHeaders($r_headers); + if(isset($this->resp_headers['set-cookie'])){ + foreach ((array) $this->resp_headers['set-cookie'] as $cookie){ + list($cookie) = explode(';',$cookie,2); + list($key,$val) = explode('=',$cookie,2); + $key = trim($key); + if($val == 'deleted'){ + if(isset($this->cookies[$key])){ + unset($this->cookies[$key]); + } + }elseif($key){ + $this->cookies[$key] = $val; } - }elseif($key){ - $this->cookies[$key] = $val; } } - } - $this->_debug('Object headers',$this->resp_headers); + $this->_debug('Object headers',$this->resp_headers); - // check server status code to follow redirect - if($this->status == 301 || $this->status == 302 ){ - // close the connection because we don't handle content retrieval here - // that's the easiest way to clean up the connection - fclose($socket); - unset(self::$connections[$connectionId]); + // check server status code to follow redirect + if($this->status == 301 || $this->status == 302 ){ + if (empty($this->resp_headers['location'])){ + throw new HTTPClientException('Redirect but no Location Header found'); + }elseif($this->redirect_count == $this->max_redirect){ + throw new HTTPClientException('Maximum number of redirects exceeded'); + }else{ + // close the connection because we don't handle content retrieval here + // that's the easiest way to clean up the connection + fclose($socket); + unset(self::$connections[$connectionId]); - if (empty($this->resp_headers['location'])){ - $this->error = 'Redirect but no Location Header found'; - return false; - }elseif($this->redirect_count == $this->max_redirect){ - $this->error = 'Maximum number of redirects exceeded'; - return false; - }else{ - $this->redirect_count++; - $this->referer = $url; - // handle non-RFC-compliant relative redirects - if (!preg_match('/^http/i', $this->resp_headers['location'])){ - if($this->resp_headers['location'][0] != '/'){ - $this->resp_headers['location'] = $uri['scheme'].'://'.$uri['host'].':'.$uri['port']. - dirname($uri['path']).'/'.$this->resp_headers['location']; - }else{ - $this->resp_headers['location'] = $uri['scheme'].'://'.$uri['host'].':'.$uri['port']. - $this->resp_headers['location']; + $this->redirect_count++; + $this->referer = $url; + // handle non-RFC-compliant relative redirects + if (!preg_match('/^http/i', $this->resp_headers['location'])){ + if($this->resp_headers['location'][0] != '/'){ + $this->resp_headers['location'] = $uri['scheme'].'://'.$uri['host'].':'.$uri['port']. + dirname($uri['path']).'/'.$this->resp_headers['location']; + }else{ + $this->resp_headers['location'] = $uri['scheme'].'://'.$uri['host'].':'.$uri['port']. + $this->resp_headers['location']; + } } + // perform redirected request, always via GET (required by RFC) + return $this->sendRequest($this->resp_headers['location'],array(),'GET'); } - // perform redirected request, always via GET (required by RFC) - return $this->sendRequest($this->resp_headers['location'],array(),'GET'); } - } - // check if headers are as expected - if($this->header_regexp && !preg_match($this->header_regexp,$r_headers)){ - $this->error = 'The received headers did not match the given regexp'; - unset(self::$connections[$connectionId]); - return false; - } + // check if headers are as expected + if($this->header_regexp && !preg_match($this->header_regexp,$r_headers)) + throw new HTTPClientException('The received headers did not match the given regexp'); - //read body (with chunked encoding if needed) - $r_body = ''; - if(preg_match('/transfer\-(en)?coding:\s*chunked\r\n/i',$r_headers)){ - do { - $chunk_size = ''; + //read body (with chunked encoding if needed) + $r_body = ''; + if(preg_match('/transfer\-(en)?coding:\s*chunked\r\n/i',$r_headers)){ + stream_set_blocking($socket,1); do { - if(feof($socket)){ - $this->error = 'Premature End of File (socket)'; - unset(self::$connections[$connectionId]); - return false; + $chunk_size = ''; + do { + if(feof($socket)) + throw new HTTPClientException('Premature End of File (socket)'); + if(time()-$start > $this->timeout) + throw new HTTPClientException(sprintf('Timeout while reading chunk (%.3fs)',$this->_time() - $this->start), -100); + $byte = fread($socket,1); + $chunk_size .= $byte; + } while (preg_match('/^[a-zA-Z0-9]?$/',$byte)); // read chunksize including \r + + $byte = fread($socket,1); // readtrailing \n + $chunk_size = hexdec($chunk_size); + if ($chunk_size) { + $read_size = $chunk_size; + while ($read_size > 0) { + $this_chunk = fread($socket,$read_size); + $r_body .= $this_chunk; + $read_size -= strlen($this_chunk); + } + $byte = fread($socket,2); // read trailing \r\n + } + + if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){ + if ($this->max_bodysize_abort) + throw new HTTPClientException('Allowed response size exceeded'); + $this->error = 'Allowed response size exceeded'; + break; } + } while ($chunk_size); + stream_set_blocking($socket,0); + }else{ + // read entire socket + while (!feof($socket)) { if(time()-$start > $this->timeout){ $this->status = -100; - $this->error = sprintf('Timeout while reading chunk (%.3fs)',$this->_time() - $this->start); + $this->error = sprintf('Timeout while reading response (%.3fs)',$this->_time() - $this->start); unset(self::$connections[$connectionId]); return false; } - $byte = fread($socket,1); - $chunk_size .= $byte; - } while (preg_match('/^[a-zA-Z0-9]?$/',$byte)); // read chunksize including \r - - $byte = fread($socket,1); // readtrailing \n - $chunk_size = hexdec($chunk_size); - if ($chunk_size) { - $read_size = $chunk_size; - while ($read_size > 0) { - $this_chunk = fread($socket,$read_size); - $r_body .= $this_chunk; - $read_size -= strlen($this_chunk); - } - $byte = fread($socket,2); // read trailing \r\n - } - - if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){ - $this->error = 'Allowed response size exceeded'; - if ($this->max_bodysize_abort){ - unset(self::$connections[$connectionId]); - return false; - } else { - break; + $r_body .= fread($socket,4096); + $r_size = strlen($r_body); + if($this->max_bodysize && $r_size > $this->max_bodysize){ + $this->error = 'Allowed response size exceeded'; + if ($this->max_bodysize_abort) { + unset(self::$connections[$connectionId]); + return false; + } else { + break; + } } - } - } while ($chunk_size); - }else{ - // read entire socket - while (!feof($socket)) { - if(time()-$start > $this->timeout){ - $this->status = -100; - $this->error = sprintf('Timeout while reading response (%.3fs)',$this->_time() - $this->start); - unset(self::$connections[$connectionId]); - return false; - } - $r_body .= fread($socket,4096); - $r_size = strlen($r_body); - if($this->max_bodysize && $r_size > $this->max_bodysize){ - $this->error = 'Allowed response size exceeded'; - if ($this->max_bodysize_abort) { - unset(self::$connections[$connectionId]); - return false; - } else { + if(isset($this->resp_headers['content-length']) && + !isset($this->resp_headers['transfer-encoding']) && + $this->resp_headers['content-length'] == $r_size){ + // we read the content-length, finish here break; } } - if(isset($this->resp_headers['content-length']) && - !isset($this->resp_headers['transfer-encoding']) && - $this->resp_headers['content-length'] == $r_size){ - // we read the content-length, finish here - break; - } } + + } catch (HTTPClientException $err) { + $this->error = $err->getMessage(); + if ($err->getCode()) + $this->status = $err->getCode(); + unset(self::$connections[$connectionId]); + fclose($socket); + return false; } if (!$this->keep_alive || -- cgit v1.2.3 From 62699eac09eab66c5c19413bf629daf9d5d6247d Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Sun, 17 Jun 2012 01:06:18 -0400 Subject: Utility functions for reading from a socket. Reading from a socket is handled by functions that encapsulate the error handling and timeout conditions. _readData reads blocks of data. _readLine reads a single line. --- inc/HTTPClient.php | 113 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 38 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index f11621f6a..ab41257bd 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -281,9 +281,6 @@ class HTTPClient { $headers['Proxy-Authorization'] = 'Basic '.base64_encode($this->proxy_user.':'.$this->proxy_pass); } - // stop time - $start = time(); - // already connected? $connectionId = $this->_uniqueConnectionId($server,$port); $this->_debug('connection pool', self::$connections); @@ -333,8 +330,9 @@ class HTTPClient { $written = 0; while($written < $towrite){ // check timeout - if(time()-$start > $this->timeout) - throw new HTTPClientException(sprintf('Timeout while sending request (%.3fs)',$this->_time() - $this->start), -100); + $time_used = $this->_time() - $this->start; + if($time_used > $this->timeout) + throw new HTTPClientException(sprintf('Timeout while sending request (%.3fs)',$time_used), -100); // wait for stream ready or timeout (1sec) if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ @@ -355,13 +353,9 @@ class HTTPClient { // read headers from socket $r_headers = ''; do{ - if(time()-$start > $this->timeout) - throw new HTTPClientException(sprintf('Timeout while reading headers (%.3fs)',$this->_time() - $this->start), -100); - if(feof($socket)) - throw new HTTPClientException('Premature End of File (socket)'); - usleep(1000); - $r_headers .= fgets($socket,1024); - }while(!preg_match('/\r?\n\r?\n$/',$r_headers)); + $r_line = $this->_readLine($socket, 'headers'); + $r_headers .= $r_line; + }while($r_line != "\r\n" && $r_line != "\n"); $this->_debug('response headers',$r_headers); @@ -436,28 +430,19 @@ class HTTPClient { //read body (with chunked encoding if needed) $r_body = ''; if(preg_match('/transfer\-(en)?coding:\s*chunked\r\n/i',$r_headers)){ - stream_set_blocking($socket,1); do { $chunk_size = ''; do { - if(feof($socket)) - throw new HTTPClientException('Premature End of File (socket)'); - if(time()-$start > $this->timeout) - throw new HTTPClientException(sprintf('Timeout while reading chunk (%.3fs)',$this->_time() - $this->start), -100); - $byte = fread($socket,1); + $byte = $this->_readData($socket, 1, 'chunk'); $chunk_size .= $byte; } while (preg_match('/^[a-zA-Z0-9]?$/',$byte)); // read chunksize including \r - - $byte = fread($socket,1); // readtrailing \n + $byte = $this->_readData($socket, 1, 'chunk'); // readtrailing \n $chunk_size = hexdec($chunk_size); + + // TODO: validate max_bodysize here to avoid the costly read if ($chunk_size) { - $read_size = $chunk_size; - while ($read_size > 0) { - $this_chunk = fread($socket,$read_size); - $r_body .= $this_chunk; - $read_size -= strlen($this_chunk); - } - $byte = fread($socket,2); // read trailing \r\n + $r_body .= $this->_readData($socket, $chunk_size, 'chunk'); + $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n } if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){ @@ -467,24 +452,16 @@ class HTTPClient { break; } } while ($chunk_size); - stream_set_blocking($socket,0); }else{ // read entire socket while (!feof($socket)) { - if(time()-$start > $this->timeout){ - $this->status = -100; - $this->error = sprintf('Timeout while reading response (%.3fs)',$this->_time() - $this->start); - unset(self::$connections[$connectionId]); - return false; - } - $r_body .= fread($socket,4096); + $r_body .= $this->_readData($socket, 0, 'response', true); $r_size = strlen($r_body); if($this->max_bodysize && $r_size > $this->max_bodysize){ - $this->error = 'Allowed response size exceeded'; if ($this->max_bodysize_abort) { - unset(self::$connections[$connectionId]); - return false; + throw new HTTPClientException('Allowed response size exceeded'); } else { + $this->error = 'Allowed response size exceeded'; break; } } @@ -532,6 +509,66 @@ class HTTPClient { return true; } + /** + * Safely read data from a socket + * + * Reads up to a given number of bytes or throws an exception if the + * response times out or ends prematurely. If the number of bytes to + * read is negative, then it will read up to the absolute value, but + * may read less. A value of 0 returns an arbitrarily sized block, + * and a positive value will return exactly that many bytes. + * + * @param handle $socket An open socket handle in non-blocking mode + * @param int $nbytes Number of bytes to read + * @param string $message Description of what is being read + * @param bool $ignore_eof End-of-file is not an error if this is set + * @author Tom N Harris + */ + function _readData($socket, $nbytes, $message, $ignore_eof = false) { + $r_data = ''; + $to_read = $nbytes ? $nbytes : 4096; + if ($to_read < 0) $to_read = -$to_read; + do { + $time_used = $this->_time() - $this->start; + if ($time_used > $this->timeout) + throw new HTTPClientException( + sprintf('Timeout while reading %s (%.3fs)', $message, $time_used), + -100); + if(!$ignore_eof && feof($socket)) + throw new HTTPClientException("Premature End of File (socket) while reading $message"); + //usleep(1000); + $bytes = fread($socket, $to_read); + $r_data .= $bytes; + $to_read -= strlen($bytes); + } while (strlen($r_data) < $nbytes); + return $r_data; + } + + /** + * Safely read a \n-terminated line from a socket + * + * Always returns a complete line, including the terminating \n. + * + * @param handle $socket An open socket handle in non-blocking mode + * @param string $message Description of what is being read + * @author Tom N Harris + */ + function _readLine($socket, $message) { + $r_data = ''; + do { + $time_used = $this->_time() - $this->start; + if ($time_used > $this->timeout) + throw new HTTPClientException( + sprintf('Timeout while reading %s (%.3fs)', $message, $time_used), + -100); + if(feof($socket)) + throw new HTTPClientException("Premature End of File (socket) while reading $message"); + usleep(1000); + $r_data .= fgets($socket, 1024); + } while (!preg_match('/\n$/',$r_data)); + return $r_data; + } + /** * print debug info * -- cgit v1.2.3 From 50d1968dceb92179b9a581e34a84d516b81511ce Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Tue, 19 Jun 2012 22:40:39 -0400 Subject: Utility function for writing to a socket --- inc/HTTPClient.php | 71 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 32 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index ab41257bd..ea9b42862 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -308,8 +308,8 @@ class HTTPClient { } try { - //set blocking - stream_set_blocking($socket,1); + //set non-blocking + stream_set_blocking($socket, false); // build request $request = "$method $request_url HTTP/".$this->http.HTTP_NL; @@ -319,36 +319,7 @@ class HTTPClient { $request .= $data; $this->_debug('request',$request); - - // select parameters - $sel_r = null; - $sel_w = array($socket); - $sel_e = null; - - // send request - $towrite = strlen($request); - $written = 0; - while($written < $towrite){ - // check timeout - $time_used = $this->_time() - $this->start; - if($time_used > $this->timeout) - throw new HTTPClientException(sprintf('Timeout while sending request (%.3fs)',$time_used), -100); - - // wait for stream ready or timeout (1sec) - if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ - usleep(1000); - continue; - } - - // write to stream - $ret = fwrite($socket, substr($request,$written,4096)); - if($ret === false) - throw new HTTPClientException('Failed writing to socket', -100); - $written += $ret; - } - - // continue non-blocking - stream_set_blocking($socket,0); + $this->_sendData($socket, $request, 'request'); // read headers from socket $r_headers = ''; @@ -509,6 +480,42 @@ class HTTPClient { return true; } + /** + * Safely write data to a socket + * + * @param handle $socket An open socket handle + * @param string $data The data to write + * @param string $message Description of what is being read + * @author Tom N Harris + */ + function _sendData($socket, $data, $message) { + // select parameters + $sel_r = null; + $sel_w = array($socket); + $sel_e = null; + + // send request + $towrite = strlen($data); + $written = 0; + while($written < $towrite){ + // check timeout + $time_used = $this->_time() - $this->start; + if($time_used > $this->timeout) + throw new HTTPClientException(sprintf('Timeout while sending %s (%.3fs)',$message, $time_used), -100); + if(feof($socket)) + throw new HTTPClientException("Socket disconnected while writing $message"); + + // wait for stream ready or timeout + if(@stream_select($sel_r, $sel_w, $sel_e, $this->timeout - $time_used) !== false){ + // write to stream + $nbytes = fwrite($socket, substr($data,$written,4096)); + if($nbytes === false) + throw new HTTPClientException("Failed writing to socket while sending $message", -100); + $written += $nbytes; + } + } + } + /** * Safely read data from a socket * -- cgit v1.2.3 From 288188afa98cd0ff65b2c2bf529fba0bff15f634 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Tue, 19 Jun 2012 23:42:00 -0400 Subject: HTTP headers are already parsed, there is no need for regexp --- inc/HTTPClient.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index ea9b42862..534ba220c 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -400,7 +400,8 @@ class HTTPClient { //read body (with chunked encoding if needed) $r_body = ''; - if(preg_match('/transfer\-(en)?coding:\s*chunked\r\n/i',$r_headers)){ + if((isset($this->resp_headers['transfer-encoding']) && $this->resp_headers['transfer-encoding'] == 'chunked') + || (isset($this->resp_headers['transfer-coding']) && $this->resp_headers['transfer-coding'] == 'chunked')){ do { $chunk_size = ''; do { -- cgit v1.2.3 From 24f112d0c2604958ac73923f758d9d37f229ff11 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 20 Jun 2012 00:52:10 -0400 Subject: Efficiently wait on sockets --- inc/HTTPClient.php | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 534ba220c..f6ab91f4f 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -507,7 +507,8 @@ class HTTPClient { throw new HTTPClientException("Socket disconnected while writing $message"); // wait for stream ready or timeout - if(@stream_select($sel_r, $sel_w, $sel_e, $this->timeout - $time_used) !== false){ + self::selecttimeout($this->timeout - $time_used, $sec, $usec); + if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ // write to stream $nbytes = fwrite($socket, substr($data,$written,4096)); if($nbytes === false) @@ -533,6 +534,11 @@ class HTTPClient { * @author Tom N Harris */ function _readData($socket, $nbytes, $message, $ignore_eof = false) { + // select parameters + $sel_r = array($socket); + $sel_w = null; + $sel_e = null; + $r_data = ''; $to_read = $nbytes ? $nbytes : 4096; if ($to_read < 0) $to_read = -$to_read; @@ -544,10 +550,16 @@ class HTTPClient { -100); if(!$ignore_eof && feof($socket)) throw new HTTPClientException("Premature End of File (socket) while reading $message"); - //usleep(1000); - $bytes = fread($socket, $to_read); - $r_data .= $bytes; - $to_read -= strlen($bytes); + + // wait for stream ready or timeout + self::selecttimeout($this->timeout - $time_used, $sec, $usec); + if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ + $bytes = fread($socket, $to_read); + if($bytes === false) + throw new HTTPClientException("Failed reading from socket while reading $message", -100); + $r_data .= $bytes; + $to_read -= strlen($bytes); + } } while (strlen($r_data) < $nbytes); return $r_data; } @@ -562,6 +574,11 @@ class HTTPClient { * @author Tom N Harris */ function _readLine($socket, $message) { + // select parameters + $sel_r = array($socket); + $sel_w = null; + $sel_e = null; + $r_data = ''; do { $time_used = $this->_time() - $this->start; @@ -571,8 +588,12 @@ class HTTPClient { -100); if(feof($socket)) throw new HTTPClientException("Premature End of File (socket) while reading $message"); - usleep(1000); - $r_data .= fgets($socket, 1024); + + // wait for stream ready or timeout + self::selecttimeout($this->timeout - $time_used, $sec, $usec); + if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ + $r_data = fgets($socket, 1024); + } } while (!preg_match('/\n$/',$r_data)); return $r_data; } @@ -597,11 +618,19 @@ class HTTPClient { /** * Return current timestamp in microsecond resolution */ - function _time(){ + static function _time(){ list($usec, $sec) = explode(" ", microtime()); return ((float)$usec + (float)$sec); } + /** + * Calculate seconds and microseconds + */ + static function selecttimeout($time, &$sec, &$usec){ + $sec = floor($time); + $usec = (int)(($time - $sec) * 1000000); + } + /** * convert given header string to Header array * -- cgit v1.2.3 From fbf63b6e769b92fc1964c09132dfc937910d04f3 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 20 Jun 2012 01:29:15 -0400 Subject: Validate the size of a chunk before reading from the socket --- inc/HTTPClient.php | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index f6ab91f4f..901c46ec9 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -404,25 +404,26 @@ class HTTPClient { || (isset($this->resp_headers['transfer-coding']) && $this->resp_headers['transfer-coding'] == 'chunked')){ do { $chunk_size = ''; - do { - $byte = $this->_readData($socket, 1, 'chunk'); + while (preg_match('/^[a-zA-Z0-9]?$/',$byte=$this->_readData($socket,1,'chunk'))){ + // read chunksize until \r $chunk_size .= $byte; - } while (preg_match('/^[a-zA-Z0-9]?$/',$byte)); // read chunksize including \r + if (strlen($chunk_size) > 128) // set an abritrary limit on the size of chunks + throw new HTTPClientException('Allowed response size exceeded'); + } $byte = $this->_readData($socket, 1, 'chunk'); // readtrailing \n $chunk_size = hexdec($chunk_size); - // TODO: validate max_bodysize here to avoid the costly read - if ($chunk_size) { - $r_body .= $this->_readData($socket, $chunk_size, 'chunk'); - $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n - } - - if($this->max_bodysize && strlen($r_body) > $this->max_bodysize){ + if($this->max_bodysize && $chunk_size+strlen($r_body) > $this->max_bodysize){ if ($this->max_bodysize_abort) throw new HTTPClientException('Allowed response size exceeded'); $this->error = 'Allowed response size exceeded'; break; } + + if ($chunk_size) { + $r_body .= $this->_readData($socket, $chunk_size, 'chunk'); + $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n + } } while ($chunk_size); }else{ // read entire socket -- cgit v1.2.3 From 769b429a77368df14e3753f624466f658e971df6 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 20 Jun 2012 03:00:18 -0400 Subject: HTTPClient will read up to max_bodysize if it can --- inc/HTTPClient.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 901c46ec9..b5e665cb1 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -402,6 +402,7 @@ class HTTPClient { $r_body = ''; if((isset($this->resp_headers['transfer-encoding']) && $this->resp_headers['transfer-encoding'] == 'chunked') || (isset($this->resp_headers['transfer-coding']) && $this->resp_headers['transfer-coding'] == 'chunked')){ + $abort = false; do { $chunk_size = ''; while (preg_match('/^[a-zA-Z0-9]?$/',$byte=$this->_readData($socket,1,'chunk'))){ @@ -417,18 +418,19 @@ class HTTPClient { if ($this->max_bodysize_abort) throw new HTTPClientException('Allowed response size exceeded'); $this->error = 'Allowed response size exceeded'; - break; + $chunk_size = $this->max_bodysize - strlen($r_body); + $abort = true; } - if ($chunk_size) { + if ($chunk_size > 0) { $r_body .= $this->_readData($socket, $chunk_size, 'chunk'); $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n } - } while ($chunk_size); + } while ($chunk_size && !$abort); }else{ // read entire socket while (!feof($socket)) { - $r_body .= $this->_readData($socket, 0, 'response', true); + $r_body .= $this->_readData($socket, -$this->max_bodysize, 'response', true); $r_size = strlen($r_body); if($this->max_bodysize && $r_size > $this->max_bodysize){ if ($this->max_bodysize_abort) { -- cgit v1.2.3 From b3b97ef358f9141bc1f1b3ebec799a0ad0771f7e Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 20 Jun 2012 03:11:57 -0400 Subject: Skip over chunk extensions that nobody uses because RFC2616 says so --- inc/HTTPClient.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index b5e665cb1..73f5b89b4 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -411,7 +411,7 @@ class HTTPClient { if (strlen($chunk_size) > 128) // set an abritrary limit on the size of chunks throw new HTTPClientException('Allowed response size exceeded'); } - $byte = $this->_readData($socket, 1, 'chunk'); // readtrailing \n + $this->_readLine($socket, 'chunk'); // readtrailing \n $chunk_size = hexdec($chunk_size); if($this->max_bodysize && $chunk_size+strlen($r_body) > $this->max_bodysize){ -- cgit v1.2.3 From 8243e61012e1d4f5614a32a3d5d9e81c50036f1c Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 27 Jun 2012 19:09:58 -0400 Subject: Limit size of reads when max_bodysize is set or content-length is present --- inc/HTTPClient.php | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 73f5b89b4..c3ccfbbf2 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -427,25 +427,25 @@ class HTTPClient { $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n } } while ($chunk_size && !$abort); + }elseif($this->max_bodysize){ + // read just over the max_bodysize + $r_body = $this->_readData($socket, $this->max_bodysize+1, 'response', true); + if(strlen($r_body) > $this->max_bodysize){ + if ($this->max_bodysize_abort) { + throw new HTTPClientException('Allowed response size exceeded'); + } else { + $this->error = 'Allowed response size exceeded'; + } + } + }elseif(isset($this->resp_headers['content-length']) && + !isset($this->resp_headers['transfer-encoding'])){ + // read up to the content-length + $r_body = $this->_readData($socket, $this->resp_headers['content-length'], 'response', true); }else{ // read entire socket + $r_size = 0; while (!feof($socket)) { - $r_body .= $this->_readData($socket, -$this->max_bodysize, 'response', true); - $r_size = strlen($r_body); - if($this->max_bodysize && $r_size > $this->max_bodysize){ - if ($this->max_bodysize_abort) { - throw new HTTPClientException('Allowed response size exceeded'); - } else { - $this->error = 'Allowed response size exceeded'; - break; - } - } - if(isset($this->resp_headers['content-length']) && - !isset($this->resp_headers['transfer-encoding']) && - $this->resp_headers['content-length'] == $r_size){ - // we read the content-length, finish here - break; - } + $r_body .= $this->_readData($socket, 0, 'response', true); } } @@ -525,10 +525,8 @@ class HTTPClient { * Safely read data from a socket * * Reads up to a given number of bytes or throws an exception if the - * response times out or ends prematurely. If the number of bytes to - * read is negative, then it will read up to the absolute value, but - * may read less. A value of 0 returns an arbitrarily sized block, - * and a positive value will return exactly that many bytes. + * response times out or ends prematurely. If $nbytes is 0, an arbitrarily + * sized block will be read. * * @param handle $socket An open socket handle in non-blocking mode * @param int $nbytes Number of bytes to read @@ -543,16 +541,19 @@ class HTTPClient { $sel_e = null; $r_data = ''; - $to_read = $nbytes ? $nbytes : 4096; - if ($to_read < 0) $to_read = -$to_read; + if ($nbytes <= 0) $nbytes = 4096; + $to_read = $nbytes; do { $time_used = $this->_time() - $this->start; if ($time_used > $this->timeout) throw new HTTPClientException( sprintf('Timeout while reading %s (%.3fs)', $message, $time_used), -100); - if(!$ignore_eof && feof($socket)) - throw new HTTPClientException("Premature End of File (socket) while reading $message"); + if(feof($socket)) { + if(!$ignore_eof) + throw new HTTPClientException("Premature End of File (socket) while reading $message"); + break; + } // wait for stream ready or timeout self::selecttimeout($this->timeout - $time_used, $sec, $usec); -- cgit v1.2.3 From a6ba0720629e19619b1d72aa7aadea28a533f856 Mon Sep 17 00:00:00 2001 From: Tom N Harris Date: Wed, 27 Jun 2012 19:38:46 -0400 Subject: Avoid timeout when content-length is 0 --- inc/HTTPClient.php | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index c3ccfbbf2..a25846c31 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -445,7 +445,7 @@ class HTTPClient { // read entire socket $r_size = 0; while (!feof($socket)) { - $r_body .= $this->_readData($socket, 0, 'response', true); + $r_body .= $this->_readData($socket, 4096, 'response', true); } } @@ -525,8 +525,7 @@ class HTTPClient { * Safely read data from a socket * * Reads up to a given number of bytes or throws an exception if the - * response times out or ends prematurely. If $nbytes is 0, an arbitrarily - * sized block will be read. + * response times out or ends prematurely. * * @param handle $socket An open socket handle in non-blocking mode * @param int $nbytes Number of bytes to read @@ -541,7 +540,8 @@ class HTTPClient { $sel_e = null; $r_data = ''; - if ($nbytes <= 0) $nbytes = 4096; + // Does not return immediately so timeout and eof can be checked + if ($nbytes < 0) $nbytes = 0; $to_read = $nbytes; do { $time_used = $this->_time() - $this->start; @@ -555,16 +555,18 @@ class HTTPClient { break; } - // wait for stream ready or timeout - self::selecttimeout($this->timeout - $time_used, $sec, $usec); - if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ - $bytes = fread($socket, $to_read); - if($bytes === false) - throw new HTTPClientException("Failed reading from socket while reading $message", -100); - $r_data .= $bytes; - $to_read -= strlen($bytes); + if ($to_read > 0) { + // wait for stream ready or timeout + self::selecttimeout($this->timeout - $time_used, $sec, $usec); + if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ + $bytes = fread($socket, $to_read); + if($bytes === false) + throw new HTTPClientException("Failed reading from socket while reading $message", -100); + $r_data .= $bytes; + $to_read -= strlen($bytes); + } } - } while (strlen($r_data) < $nbytes); + } while ($to_read > 0 && strlen($r_data) < $nbytes); return $r_data; } -- cgit v1.2.3