From 11a4aba9a63c82b0bb7805b50b8a761cc5a9c493 Mon Sep 17 00:00:00 2001 From: Steven Wittens Date: Mon, 25 Jul 2005 20:40:35 +0000 Subject: - #26688: Add mbstring support to Drupal and clear up string handling fuzzies. --- includes/common.inc | 4 +- includes/unicode.inc | 446 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 includes/unicode.inc (limited to 'includes') diff --git a/includes/common.inc b/includes/common.inc index 1c7aa0a7c..93724afd6 100644 --- a/includes/common.inc +++ b/includes/common.inc @@ -1831,7 +1831,7 @@ function drupal_implode_autocomplete($array) { function _drupal_bootstrap_full() { static $called; - global $locale; + global $locale, $multibyte; if ($called) { return; @@ -1849,6 +1849,8 @@ function _drupal_bootstrap_full() { set_error_handler('error_handler'); // Emit the correct charset HTTP header. drupal_set_header('Content-Type: text/html; charset=utf-8'); + // Detect string handling method + $multibyte = unicode_check(); // Initialize $_GET['q'] prior to loading modules and invoking hook_init(). if (!empty($_GET['q'])) { $_GET['q'] = drupal_get_normal_path(trim($_GET['q'], '/')); diff --git a/includes/unicode.inc b/includes/unicode.inc new file mode 100644 index 000000000..368bc8596 --- /dev/null +++ b/includes/unicode.inc @@ -0,0 +1,446 @@ +PHP PCRE documentation for more information.', array('%url' => 'http://be.php.net/manual/nl/ref.pcre.php'))); + } + return UNICODE_ERROR; + } + + // Check for mbstring extension + if (!function_exists('mb_strlen')) { + return UNICODE_SINGLEBYTE; + } + + // Check mbstring configuration + if (ini_get('mbstring.func_overload') != 0) { + if ($errors) { + form_set_error('unicode', t('Multibyte string function overloading in PHP is active and must be disabled. Check the php.ini mbstring.func_overload setting. Please refer to the PHP mbstring documentation for more information.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php'))); + } + return UNICODE_ERROR; + } + if (ini_get('mbstring.encoding_translation') != 0) { + if ($errors) { + form_set_error('unicode', t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini mbstring.encoding_translation setting. Please refer to the PHP mbstring documentation for more information.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php'))); + } + return UNICODE_ERROR; + } + if (ini_get('mbstring.http_input') != 'pass') { + if ($errors) { + form_set_error('unicode', t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini mbstring.http_input setting. Please refer to the PHP mbstring documentation for more information.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php'))); + } + return UNICODE_ERROR; + } + if (ini_get('mbstring.http_output') != 'pass') { + if ($errors) { + form_set_error('unicode', t('Multibyte string output conversion in PHP is active and must be disabled. Check the php.ini mbstring.http_output setting. Please refer to the PHP mbstring documentation for more information.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php'))); + } + return UNICODE_ERROR; + } + + // Set appropriate configuration + mb_internal_encoding('utf-8'); + mb_language('uni'); + return UNICODE_MULTIBYTE; +} + +/** + * Return the required Unicode status and errors for admin/settings. + */ +function unicode_settings() { + $status = unicode_check(true); + $options = array(UNICODE_SINGLEBYTE => t('Standard PHP: operations on Unicode strings are emulated on a best-effort basis. Install the PHP mbstring extension for improved Unicode support.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php')), + UNICODE_MULTIBYTE => t('Multi-byte: operations on Unicode strings are supported through the PHP mbstring extension.', array('%url' => 'http://www.php.net/manual/nl/ref.mbstring.php')), + UNICODE_ERROR => t('Invalid: the current configuration is incompatible with Drupal.')); + return form_item(t('String handling method'), $options[$status]); +} + +/** + * Prepare a new XML parser. + * + * This is a wrapper around xml_parser_create() which extracts the encoding from + * the XML data first and sets the output encoding to UTF-8. This function should + * be used instead of xml_parser_create(), because PHP's XML parser doesn't check + * the input encoding itself. + * + * This is also where unsupported encodings will be converted. + * Callers should take this into account: $data might have been changed after + * the call. + * + * @param &$data + * The XML data which will be parsed later. + * @return + * An XML parser object. + */ +function drupal_xml_parser_create(&$data) { + // Default XML encoding is UTF-8 + $encoding = 'utf-8'; + $bom = false; + + // Check for UTF-8 byte order mark (PHP5's XML parser doesn't handle it). + if (!strncmp($data, "\xEF\xBB\xBF", 3)) { + $bom = true; + $data = substr($data, 3); + } + + // Check for an encoding declaration in the XML prolog if no BOM was found. + if (!$bom && ereg('^<\?xml[^>]+encoding="([^"]+)"', $data, $match)) { + $encoding = $match[1]; + } + + // Unsupported encodings are converted here into UTF-8. + $php_supported = array('utf-8', 'iso-8859-1', 'us-ascii'); + if (!in_array(strtolower($encoding), $php_supported)) { + $out = drupal_convert_to_utf8($data, $encoding); + if ($out !== false) { + $data = $out; + $encoding = 'utf-8'; + } + else { + watchdog('php', t("Could not convert XML encoding '%s' to UTF-8.", array('%s' => $encoding)), WATCHDOG_WARNING); + return 0; + } + } + + $xml_parser = xml_parser_create($encoding); + xml_parser_set_option($xml_parser, XML_OPTION_TARGET_ENCODING, 'utf-8'); + return $xml_parser; +} + +/** + * Convert data to UTF-8 + * + * Requires the iconv, GNU recode or mbstring PHP extension. + * + * @param $data + * The data to be converted. + * @param $encoding + * The encoding that the data is in + * @return + * Converted data or FALSE. + */ +function drupal_convert_to_utf8($data, $encoding) { + if (function_exists('iconv')) { + $out = @iconv($encoding, 'utf-8', $data); + } + else if (function_exists('mb_convert_encoding')) { + $out = @mb_convert_encoding($data, 'utf-8', $encoding); + } + else if (function_exists('recode_string')) { + $out = @recode_string($encoding .'..utf-8', $data); + } + else { + watchdog('php', t("Unsupported encoding '%s'. Please install iconv, GNU recode or mbstring for PHP.", array('%s' => $encoding)), WATCHDOG_ERROR); + return FALSE; + } + + return $out; +} + +/** + * Truncate a UTF-8-encoded string safely to a number of bytes. + * + * If the end position is in the middle of a UTF-8 sequence, it scans backwards + * until the beginning of the byte sequence. + * + * Use this function whenever you want to chop off a string at an unsure + * location. On the other hand, if you're sure that you're splitting on a + * character boundary (e.g. after using strpos() or similar), you can safely use + * substr() instead. + * + * @param $string + * The string to truncate. + * @param $len + * An upper limit on the returned string length. + * @param $wordsafe + * Flag to truncate at nearest space. Defaults to FALSE. + * @return + * The truncated string. + */ +function truncate_utf8($string, $len, $wordsafe = FALSE, $dots = FALSE) { + $slen = strlen($string); + if ($slen <= $len) { + return $string; + } + if ($wordsafe) { + while (($string{--$len} != ' ') && ($len > 0)) {}; + } + if ((ord($string{$len}) < 0x80) || (ord($string{$len}) >= 0xC0)) { + return substr($string, 0, $len) . ($dots ? ' ...' : ''); + } + while (ord($string{--$len}) < 0xC0) {}; + return substr($string, 0, $len) . ($dots ? ' ...' : ''); +} + +/** + * Encodes MIME/HTTP header values that contain non-ASCII, UTF-8 encoded + * characters. + * + * For example, mime_header_encode('tést.txt') returns "=?UTF-8?B?dMOpc3QudHh0?=". + * + * See http://www.rfc-editor.org/rfc/rfc2047.txt for more information. + * + * Notes: + * - Only encode strings that contain non-ASCII characters. + * - We progressively cut-off a chunk with truncate_utf8(). This is to ensure + * each chunk starts and ends on a character boundary. + * - Using \n as the chunk separator may cause problems on some systems and may + * have to be changed to \r\n or \r. + */ +function mime_header_encode($string) { + if (!preg_match('/^[\x20-\x7E]*$/', $string)) { + $chunk_size = 47; // floor((75 - strlen("=?UTF-8?B??=")) * 0.75); + $len = strlen($string); + $output = ''; + while ($len > 0) { + $chunk = truncate_utf8($string, $chunk_size); + $output .= ' =?UTF-8?B?'. base64_encode($chunk) ."?=\n"; + $c = strlen($chunk); + $string = substr($string, $c); + $len -= $c; + } + return trim($output); + } + return $string; +} + +/** + * Decode all HTML entities (including numerical ones) to regular UTF-8 bytes. + * Double-escaped entities will only be decoded once ("&lt;" becomes "<", not "<"). + * + * @param $text + * The text to decode entities in. + * @param $exclude + * An array of characters which should not be decoded. For example, + * array('<', '&', '"'). This affects both named and numerical entities. + */ +function decode_entities($text, $exclude = array()) { + static $table; + // We store named entities in a table for quick processing. + if (!isset($table)) { + // Get all named HTML entities. + $table = array_flip(get_html_translation_table(HTML_ENTITIES)); + // PHP gives us ISO-8859-1 data, we need UTF-8. + $table = array_map('utf8_encode', $table); + // Add apostrophe (XML) + $table['''] = "'"; + } + $newtable = array_diff($table, $exclude); + + // Use a regexp to select all entities in one pass, to avoid decoding double-escaped entities twice. + return preg_replace('/&(#x?)?([A-Za-z0-9]+);/e', '_decode_entities("$1", "$2", "$0", $newtable, $exclude)', $text); +} + +/** + * Helper function for decode_entities + */ +function _decode_entities($prefix, $codepoint, $original, &$table, &$exclude) { + // Named entity + if (!$prefix) { + if (isset($table[$original])) { + return $table[$original]; + } + else { + return $original; + } + } + // Hexadecimal numerical entity + if ($prefix == '#x') { + $codepoint = base_convert($codepoint, 16, 10); + } + // Encode codepoint as UTF-8 bytes + if ($codepoint < 0x80) { + $str = chr($codepoint); + } + else if ($codepoint < 0x800) { + $str = chr(0xC0 | ($codepoint >> 6)) + . chr(0x80 | ($codepoint & 0x3F)); + } + else if ($codepoint < 0x10000) { + $str = chr(0xE0 | ( $codepoint >> 12)) + . chr(0x80 | (($codepoint >> 6) & 0x3F)) + . chr(0x80 | ( $codepoint & 0x3F)); + } + else if ($codepoint < 0x200000) { + $str = chr(0xF0 | ( $codepoint >> 18)) + . chr(0x80 | (($codepoint >> 12) & 0x3F)) + . chr(0x80 | (($codepoint >> 6) & 0x3F)) + . chr(0x80 | ( $codepoint & 0x3F)); + } + // Check for excluded characters + if (in_array($str, $exclude)) { + return $original; + } + else { + return $str; + } +} + +/** + * Count the amount of characters in a UTF-8 string. This is less than or + * equal to the byte count. + */ +function drupal_strlen($text) { + global $multibyte; + if ($multibyte == UNICODE_MULTIBYTE) { + return mb_strlen($text); + } + else { + // Do not count UTF-8 continuation bytes. + return strlen(preg_replace("/[\x80-\xBF]/", '', $text)); + } +} + +/** + * Uppercase a UTF-8 string. + */ +function drupal_strtoupper($text) { + global $multibyte; + if ($multibyte == UNICODE_MULTIBYTE) { + //return mb_convert_case($text, MB_CASE_UPPER, 'utf-8'); + return mb_strtoupper($text); + } + else { + // Use C-locale for ASCII-only uppercase + $text = strtoupper($text); + // Case flip Latin-1 accented letters + $text = preg_replace_callback('/\xC3[\xA0-\xB6\xB8-\xBE]/', '_unicode_caseflip', $text); + return $text; + } +} + +/** + * Lowercase a UTF-8 string. + */ +function drupal_strtolower($text) { + global $multibyte; + if ($multibyte == UNICODE_MULTIBYTE) { + return mb_strtolower($text); + } + else { + // Use C-locale for ASCII-only lowercase + $text = strtolower($text); + // Case flip Latin-1 accented letters + $text = preg_replace_callback('/\xC3[\x80-\x96\x98-\x9E]/', '_unicode_caseflip', $text); + return $text; + } +} + +/** + * Helper function for case conversion of Latin-1. + * Used for flipping U+C0-U+DE to U+E0-U+FD and back. + */ +function _unicode_caseflip($matches) { + return $matches[0]{0} . chr(ord($matches[0]{1}) ^ 32); +} + +/** + * Capitalize the first letter of a UTF-8 string. + */ +function drupal_ucfirst($text) { + // Note: no mbstring equivalent! + return drupal_strtoupper(drupal_substr($text, 0, 1)) . drupal_substr($text, 1); +} + +/** + * Cut off a piece of a string based on character indices and counts. Follows + * the same behaviour as PHP's own substr() function. + * + * Note that for cutting off a string at a known character/substring + * location, the usage of PHP's normal strpos/substr is safe and + * much faster. + */ +function drupal_substr($text, $start, $length = NULL) { + global $multibyte; + if ($multibyte == UNICODE_MULTIBYTE) { + return $length === NULL ? mb_substr($text, $start) : mb_substr($text, $start, $length); + } + else { + $strlen = strlen($text); + // Find the starting byte offset + if ($start > 0) { + // Count all the continuation bytes from the start until we have found + // $start characters + $bytes = -1; $chars = -1; + while ($bytes < $strlen && $chars < $start) { + $bytes++; + $c = ord($text{$bytes}); + if ($c < 0x80 || $c >= 0xC0) { + $chars++; + } + } + } + else if ($start < 0) { + // Count all the continuation bytes from the end until we have found + // abs($start) characters + $start = abs($start); + $bytes = $strlen; $chars = 0; + while ($bytes > 0 && $chars < $start) { + $bytes--; + $c = ord($text{$bytes}); + if ($c < 0x80 || $c >= 0xC0) { + $chars++; + } + } + } + $istart = $bytes; + + // Find the ending byte offset + if ($length === NULL) { + $bytes = $strlen - 1; + } + else if ($length > 0) { + // Count all the continuation bytes from the starting index until we have + // found $length + 1 characters. Then backtrack one byte. + $bytes = $istart; $chars = 0; + while ($bytes < $strlen && $chars < $length) { + $bytes++; + $c = ord($text{$bytes}); + if ($c < 0x80 || $c >= 0xC0) { + $chars++; + } + } + $bytes--; + } + else if ($length < 0) { + // Count all the continuation bytes from the end until we have found + // abs($length) characters + $length = abs($length); + $bytes = $strlen - 1; $chars = 0; + while ($bytes >= 0 && $chars < $length) { + $c = ord($text{$bytes}); + if ($c < 0x80 || $c >= 0xC0) { + $chars++; + } + $bytes--; + } + } + $iend = $bytes; + + return substr($text, $istart, max(0, $iend - $istart + 1)); + } +} + +?> \ No newline at end of file -- cgit v1.2.3