From 1ea7a6bada66fc9b7a45f61b4892e4ea23196d89 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sat, 28 Jul 2012 14:55:45 +0200 Subject: support CONNECT for tunneling SSL via HTTP proxies FS#2431 The included test cases currently expect a squid at localhost:3128 --- inc/HTTPClient.php | 66 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 5 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index a25846c31..8d2c10be1 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -254,11 +254,7 @@ class HTTPClient { if(!empty($uri['port'])) $headers['Host'].= ':'.$uri['port']; $headers['User-Agent'] = $this->agent; $headers['Referer'] = $this->referer; - if ($this->keep_alive) { - $headers['Connection'] = 'Keep-Alive'; - } else { - $headers['Connection'] = 'Close'; - } + if($method == 'POST'){ if(is_array($data)){ if($headers['Content-Type'] == 'multipart/form-data'){ @@ -299,6 +295,14 @@ class HTTPClient { return false; } + // try establish a CONNECT tunnel for SSL + if($this->_ssltunnel($socket, $request_url)){ + // no keep alive for tunnels + $this->keep_alive = false; + // tunnel is authed already + if(isset($headers['Proxy-Authentication'])) unset($headers['Proxy-Authentication']); + } + // keep alive? if ($this->keep_alive) { self::$connections[$connectionId] = $socket; @@ -307,6 +311,15 @@ class HTTPClient { } } + if ($this->keep_alive && !$this->proxy_host) { + // RFC 2068, section 19.7.1: A client MUST NOT send the Keep-Alive + // connection token to a proxy server. We still do keep the connection the + // proxy alive (well except for CONNECT tunnels) + $headers['Connection'] = 'Keep-Alive'; + } else { + $headers['Connection'] = 'Close'; + } + try { //set non-blocking stream_set_blocking($socket, false); @@ -484,6 +497,49 @@ class HTTPClient { return true; } + /** + * Tries to establish a CONNECT tunnel via Proxy + * + * Protocol, Servername and Port will be stripped from the request URL when a successful CONNECT happened + * + * @param ressource &$socket + * @param string &$requesturl + * @return bool true if a tunnel was established + */ + function _ssltunnel(&$socket, &$requesturl){ + if(!$this->proxy_host) return false; + $requestinfo = parse_url($requesturl); + if($requestinfo['scheme'] != 'https') return false; + if(!$requestinfo['port']) $requestinfo['port'] = 443; + + // build request + $request = "CONNECT {$requestinfo['host']}:{$requestinfo['port']} HTTP/1.0".HTTP_NL; + $request .= "Host: {$requestinfo['host']}".HTTP_NL; + if($this->proxy_user) { + 'Proxy-Authorization Basic '.base64_encode($this->proxy_user.':'.$this->proxy_pass).HTTP_NL; + } + $request .= HTTP_NL; + + $this->_debug('SSL Tunnel CONNECT',$request); + $this->_sendData($socket, $request, 'SSL Tunnel CONNECT'); + + // read headers from socket + $r_headers = ''; + do{ + $r_line = $this->_readLine($socket, 'headers'); + $r_headers .= $r_line; + }while($r_line != "\r\n" && $r_line != "\n"); + + $this->_debug('SSL Tunnel Response',$r_headers); + if(preg_match('/^HTTP\/1\.0 200/i',$r_headers)){ + if (stream_socket_enable_crypto($socket, true, STREAM_CRYPTO_METHOD_SSLv3_CLIENT)) { + $requesturl = $requestinfo['path']; + return true; + } + } + return false; + } + /** * Safely write data to a socket * -- cgit v1.2.3 From bfd975d26ab51152ac6a256827ffda93b15df48b Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Tue, 6 Nov 2012 20:58:38 +0100 Subject: fix regression bug in HTTPClient FS#2621 In the recent refactoring of the HTTPClient, a problem with certain systems was reintroduced. On these systems a select() call always waits for a timeout on the first call before working properly on the second call. This patch reintroduces the shorter timeouts with usleep rate limiting again. Since this bug is not reproducible on other systems it can't be unit tested unfortunately. --- inc/HTTPClient.php | 54 ++++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index a25846c31..c4cfcbf7c 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -509,15 +509,17 @@ class HTTPClient { if(feof($socket)) throw new HTTPClientException("Socket disconnected while writing $message"); - // wait for stream ready or timeout - self::selecttimeout($this->timeout - $time_used, $sec, $usec); - if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ - // write to stream - $nbytes = fwrite($socket, substr($data,$written,4096)); - if($nbytes === false) - throw new HTTPClientException("Failed writing to socket while sending $message", -100); - $written += $nbytes; + // wait for stream ready or timeout (1sec) + if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ + usleep(1000); + continue; } + + // write to stream + $nbytes = fwrite($socket, substr($data,$written,4096)); + if($nbytes === false) + throw new HTTPClientException("Failed writing to socket while sending $message", -100); + $written += $nbytes; } } @@ -556,15 +558,17 @@ class HTTPClient { } if ($to_read > 0) { - // wait for stream ready or timeout - self::selecttimeout($this->timeout - $time_used, $sec, $usec); - if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ - $bytes = fread($socket, $to_read); - if($bytes === false) - throw new HTTPClientException("Failed reading from socket while reading $message", -100); - $r_data .= $bytes; - $to_read -= strlen($bytes); + // wait for stream ready or timeout (1sec) + if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ + usleep(1000); + continue; } + + $bytes = fread($socket, $to_read); + if($bytes === false) + throw new HTTPClientException("Failed reading from socket while reading $message", -100); + $r_data .= $bytes; + $to_read -= strlen($bytes); } } while ($to_read > 0 && strlen($r_data) < $nbytes); return $r_data; @@ -595,11 +599,13 @@ class HTTPClient { if(feof($socket)) throw new HTTPClientException("Premature End of File (socket) while reading $message"); - // wait for stream ready or timeout - self::selecttimeout($this->timeout - $time_used, $sec, $usec); - if(@stream_select($sel_r, $sel_w, $sel_e, $sec, $usec) !== false){ - $r_data = fgets($socket, 1024); + // wait for stream ready or timeout (1sec) + if(@stream_select($sel_r,$sel_w,$sel_e,1) === false){ + usleep(1000); + continue; } + + $r_data = fgets($socket, 1024); } while (!preg_match('/\n$/',$r_data)); return $r_data; } @@ -629,14 +635,6 @@ class HTTPClient { return ((float)$usec + (float)$sec); } - /** - * Calculate seconds and microseconds - */ - static function selecttimeout($time, &$sec, &$usec){ - $sec = floor($time); - $usec = (int)(($time - $sec) * 1000000); - } - /** * convert given header string to Header array * -- cgit v1.2.3 From 78361632ede7770a5ee42b88bed0a9506b1fcb66 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sat, 16 Feb 2013 11:52:05 +0100 Subject: better HTTPClient debug output on CLI --- inc/HTTPClient.php | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 51c1de875..8cfb86567 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -669,10 +669,26 @@ class HTTPClient { /** * print debug info * + * Uses _debug_text or _debug_html depending on the SAPI name + * * @author Andreas Gohr */ function _debug($info,$var=null){ if(!$this->debug) return; + if(php_sapi_name() == 'cli'){ + $this->_debug_text($info, $var); + }else{ + $this->_debug_html($info, $var); + } + } + + /** + * print debug info as HTML + * + * @param $info + * @param null $var + */ + function _debug_html($info, $var=null){ print ''.$info.' '.($this->_time() - $this->start).'s
'; if(!is_null($var)){ ob_start(); @@ -683,6 +699,18 @@ class HTTPClient { } } + /** + * prints debug info as plain text + * + * @param $info + * @param null $var + */ + function _debug_text($info, $var=null){ + print '*'.$info.'* '.($this->_time() - $this->start)."s\n"; + if(!is_null($var)) print_r($var); + print "\n-----------------------------------------------\n"; + } + /** * Return current timestamp in microsecond resolution */ -- cgit v1.2.3 From 0fa584c26e2ba6aec1045f55551b2b1865a9fd96 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sat, 16 Feb 2013 13:07:27 +0100 Subject: HTTPClient: fixed max_bodysize when using keep-alive --- inc/HTTPClient.php | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 8cfb86567..e36514286 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -215,6 +215,9 @@ class HTTPClient { $this->start = $this->_time(); $this->error = ''; $this->status = 0; + $this->status = 0; + $this->resp_body = ''; + $this->resp_headers = array(); // don't accept gzip if truncated bodies might occur if($this->max_bodysize && @@ -440,9 +443,31 @@ class HTTPClient { $byte = $this->_readData($socket, 2, 'chunk'); // read trailing \r\n } } while ($chunk_size && !$abort); - }elseif($this->max_bodysize){ - // read just over the max_bodysize - $r_body = $this->_readData($socket, $this->max_bodysize+1, 'response', true); + }elseif(isset($this->resp_headers['content-length']) && !isset($this->resp_headers['transfer-encoding'])){ + /* RFC 2616 + * If a message is received with both a Transfer-Encoding header field and a Content-Length + * header field, the latter MUST be ignored. + */ + + // read up to the content-length or max_bodysize + // for keep alive we need to read the whole message to clean up the socket for the next read + if(!$this->keep_alive && $this->max_bodysize && $this->max_bodysize < $this->resp_headers['content-length']){ + $length = $this->max_bodysize; + }else{ + $length = $this->resp_headers['content-length']; + } + + $r_body = $this->_readData($socket, $length, 'response (content-length limited)', true); + }else{ + // read entire socket + $r_size = 0; + while (!feof($socket)) { + $r_body .= $this->_readData($socket, 4096, 'response (unlimited)', true); + } + } + + // recheck body size, we might had to read the whole body, so we abort late or trim here + if($this->max_bodysize){ if(strlen($r_body) > $this->max_bodysize){ if ($this->max_bodysize_abort) { throw new HTTPClientException('Allowed response size exceeded'); @@ -450,16 +475,6 @@ class HTTPClient { $this->error = 'Allowed response size exceeded'; } } - }elseif(isset($this->resp_headers['content-length']) && - !isset($this->resp_headers['transfer-encoding'])){ - // read up to the content-length - $r_body = $this->_readData($socket, $this->resp_headers['content-length'], 'response', true); - }else{ - // read entire socket - $r_size = 0; - while (!feof($socket)) { - $r_body .= $this->_readData($socket, 4096, 'response', true); - } } } catch (HTTPClientException $err) { -- cgit v1.2.3 From 7b9186a5c6e5a5f070396dd6455e8b06b9da16c8 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sat, 16 Feb 2013 13:14:38 +0100 Subject: HTTPClient: updated function comments --- inc/HTTPClient.php | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index e36514286..6f00d6d91 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -150,6 +150,7 @@ class HTTPClient { * * @param string $url The URL to fetch * @param bool $sloppy304 Return body on 304 not modified + * @return bool|string response body, false on error * @author Andreas Gohr */ function get($url,$sloppy304=false){ @@ -170,6 +171,7 @@ class HTTPClient { * @param string $url The URL to fetch * @param array $data Associative array of parameters * @param bool $sloppy304 Return body on 304 not modified + * @return bool|string response body, false on error * @author Andreas Gohr */ function dget($url,$data,$sloppy304=false){ @@ -187,6 +189,9 @@ class HTTPClient { * * Returns the resulting page or false on an error; * + * @param string $url The URL to fetch + * @param array $data Associative array of parameters + * @return bool|string response body, false on error * @author Andreas Gohr */ function post($url,$data){ @@ -517,8 +522,8 @@ class HTTPClient { * * Protocol, Servername and Port will be stripped from the request URL when a successful CONNECT happened * - * @param ressource &$socket - * @param string &$requesturl + * @param resource &$socket + * @param string &$requesturl * @return bool true if a tunnel was established */ function _ssltunnel(&$socket, &$requesturl){ @@ -558,9 +563,10 @@ class HTTPClient { /** * Safely write data to a socket * - * @param handle $socket An open socket handle - * @param string $data The data to write - * @param string $message Description of what is being read + * @param resource $socket An open socket handle + * @param string $data The data to write + * @param string $message Description of what is being read + * @throws HTTPClientException * @author Tom N Harris */ function _sendData($socket, $data, $message) { @@ -600,10 +606,12 @@ class HTTPClient { * Reads up to a given number of bytes or throws an exception if the * response times out or ends prematurely. * - * @param handle $socket An open socket handle in non-blocking mode - * @param int $nbytes Number of bytes to read - * @param string $message Description of what is being read - * @param bool $ignore_eof End-of-file is not an error if this is set + * @param resource $socket An open socket handle in non-blocking mode + * @param int $nbytes Number of bytes to read + * @param string $message Description of what is being read + * @param bool $ignore_eof End-of-file is not an error if this is set + * @throws HTTPClientException + * @return string * @author Tom N Harris */ function _readData($socket, $nbytes, $message, $ignore_eof = false) { @@ -650,8 +658,10 @@ class HTTPClient { * * Always returns a complete line, including the terminating \n. * - * @param handle $socket An open socket handle in non-blocking mode - * @param string $message Description of what is being read + * @param resource $socket An open socket handle in non-blocking mode + * @param string $message Description of what is being read + * @throws HTTPClientException + * @return string * @author Tom N Harris */ function _readLine($socket, $message) { @@ -840,6 +850,8 @@ class HTTPClient { /** * Generates a unique identifier for a connection. * + * @param string $server + * @param string $port * @return string unique identifier */ function _uniqueConnectionId($server, $port) { -- cgit v1.2.3 From 8b48b31f60c1acd63fbe9f35207377209faac012 Mon Sep 17 00:00:00 2001 From: Andreas Gohr Date: Sat, 16 Feb 2013 13:17:40 +0100 Subject: HTTPclient: print number of received bytes on error --- inc/HTTPClient.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'inc/HTTPClient.php') diff --git a/inc/HTTPClient.php b/inc/HTTPClient.php index 6f00d6d91..772b580b2 100644 --- a/inc/HTTPClient.php +++ b/inc/HTTPClient.php @@ -628,8 +628,8 @@ class HTTPClient { $time_used = $this->_time() - $this->start; if ($time_used > $this->timeout) throw new HTTPClientException( - sprintf('Timeout while reading %s (%.3fs)', $message, $time_used), - -100); + sprintf('Timeout while reading %s after %d bytes (%.3fs)', $message, + strlen($r_data), $time_used), -100); if(feof($socket)) { if(!$ignore_eof) throw new HTTPClientException("Premature End of File (socket) while reading $message"); -- cgit v1.2.3