Truncate UTF-8 patch. Introduced a new function truncate_utf8() for chopping off strings at unsure locations, without risking incomplete UTF-8 data.

9 files changed, 36 insertions, 11 deletions

diff --git a/includes/common.inc b/includes/common.inc
index 93d964eed..4433791b8 100644
--- a/includes/common.inc
+++ b/includes/common.inc
@@ -916,7 +916,7 @@ function format_name($object) {
     */
 
     if (strlen($object->name) > 20) {
-      $name = substr($object->name, 0, 15) ."...";
+      $name = truncate_utf8($object->name, 15) ."...";
     }
     else {
       $name = $object->name;
@@ -1234,6 +1234,31 @@ function drupal_xml_parser_create(&$data) {
   return $xml_parser;
 }
 
+/**
+ * UTF-8-safe string truncation
+ * If the end position is in the middle of a UTF-8 sequence, it scans backwards
+ * until the beginning of the byte sequence.
+ *
+ * Use this function whenever you want to chop off a string at an unsure
+ * location. On the other hand, if you're sure that you're splitting on a
+ * character boundary (e.g. after using strpos or similar), you can safely use
+ * substr() instead.
+ *
+ * @param $string The string to truncate
+ * @param $len An upper limit on the returned string length.
+ */
+function truncate_utf8($string, $len) {
+  $slen = strlen($string);
+  if ($slen <= $len) {
+    return $string;
+  }
+  if ((ord($string[$len]) < 0x80) || (ord($string[$len]) >= 0xC0)) {
+    return substr($string, 0, $len);
+  }
+  while (ord($string[--$len]) < 0xC0) {};
+  return substr($string, 0, $len);
+}
+
 include_once "includes/theme.inc";
 include_once "includes/pager.inc";
 include_once "includes/menu.inc";
diff --git a/modules/aggregator.module b/modules/aggregator.module
index 331387cfa..8b78b71fd 100644
--- a/modules/aggregator.module
+++ b/modules/aggregator.module
@@ -397,7 +397,7 @@ function aggregator_parse_feed(&$data, $feed) {
       $title = $item["TITLE"];
     }
     else {
-      $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", substr($item["DESCRIPTION"], 0, 40));
+      $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item["DESCRIPTION"], 40));
     }
 
     /*
diff --git a/modules/aggregator/aggregator.module b/modules/aggregator/aggregator.module
index 331387cfa..8b78b71fd 100644
--- a/modules/aggregator/aggregator.module
+++ b/modules/aggregator/aggregator.module
@@ -397,7 +397,7 @@ function aggregator_parse_feed(&$data, $feed) {
       $title = $item["TITLE"];
     }
     else {
-      $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", substr($item["DESCRIPTION"], 0, 40));
+      $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item["DESCRIPTION"], 40));
     }
 
     /*
diff --git a/modules/comment.module b/modules/comment.module
index 03f6144bd..24e2a232b 100644
--- a/modules/comment.module
+++ b/modules/comment.module
@@ -277,7 +277,7 @@ function comment_post($edit) {
     $edit["subject"] = strip_tags($edit["subject"]);
 
     if ($edit["subject"] == "") {
-      $edit["subject"] = substr(strip_tags($edit["comment"]), 0, 29);
+      $edit["subject"] = truncate_utf8(strip_tags($edit["comment"]), 29);
     }
 
     /*
@@ -978,7 +978,7 @@ function comment_admin_overview($status = 0) {
   $result = pager_query($sql,  50);
 
   while ($comment = db_fetch_object($result)) {
-    $rows[] = array(l($comment->subject, "node/view/$comment->nid/$comment->cid", array("title" => htmlspecialchars(substr($comment->comment, 0, 128))), NULL, "comment-$comment->cid") ." ". (node_is_new($comment->nid, $comment->timestamp) ? theme("mark") : ""), format_name($comment), ($comment->status == 0 ? t("published") : t("not published")) ."</td><td>". format_date($comment->timestamp, "small") ."</td><td>". l(t("edit comment"), "admin/comment/edit/$comment->cid"), l(t("delete comment"), "admin/comment/delete/$comment->cid"));
+    $rows[] = array(l($comment->subject, "node/view/$comment->nid/$comment->cid", array("title" => htmlspecialchars(truncate_utf8($comment->comment, 128))), NULL, "comment-$comment->cid") ." ". (node_is_new($comment->nid, $comment->timestamp) ? theme("mark") : ""), format_name($comment), ($comment->status == 0 ? t("published") : t("not published")) ."</td><td>". format_date($comment->timestamp, "small") ."</td><td>". l(t("edit comment"), "admin/comment/edit/$comment->cid"), l(t("delete comment"), "admin/comment/delete/$comment->cid"));
   }
 
   if ($pager = theme("pager", NULL, 50, 0, tablesort_pager())) {
diff --git a/modules/comment/comment.module b/modules/comment/comment.module
index 03f6144bd..24e2a232b 100644
--- a/modules/comment/comment.module
+++ b/modules/comment/comment.module
@@ -277,7 +277,7 @@ function comment_post($edit) {
     $edit["subject"] = strip_tags($edit["subject"]);
 
     if ($edit["subject"] == "") {
-      $edit["subject"] = substr(strip_tags($edit["comment"]), 0, 29);
+      $edit["subject"] = truncate_utf8(strip_tags($edit["comment"]), 29);
     }
 
     /*
@@ -978,7 +978,7 @@ function comment_admin_overview($status = 0) {
   $result = pager_query($sql,  50);
 
   while ($comment = db_fetch_object($result)) {
-    $rows[] = array(l($comment->subject, "node/view/$comment->nid/$comment->cid", array("title" => htmlspecialchars(substr($comment->comment, 0, 128))), NULL, "comment-$comment->cid") ." ". (node_is_new($comment->nid, $comment->timestamp) ? theme("mark") : ""), format_name($comment), ($comment->status == 0 ? t("published") : t("not published")) ."</td><td>". format_date($comment->timestamp, "small") ."</td><td>". l(t("edit comment"), "admin/comment/edit/$comment->cid"), l(t("delete comment"), "admin/comment/delete/$comment->cid"));
+    $rows[] = array(l($comment->subject, "node/view/$comment->nid/$comment->cid", array("title" => htmlspecialchars(truncate_utf8($comment->comment, 128))), NULL, "comment-$comment->cid") ." ". (node_is_new($comment->nid, $comment->timestamp) ? theme("mark") : ""), format_name($comment), ($comment->status == 0 ? t("published") : t("not published")) ."</td><td>". format_date($comment->timestamp, "small") ."</td><td>". l(t("edit comment"), "admin/comment/edit/$comment->cid"), l(t("delete comment"), "admin/comment/delete/$comment->cid"));
   }
 
   if ($pager = theme("pager", NULL, 50, 0, tablesort_pager())) {
diff --git a/modules/node.module b/modules/node.module
index c9392f776..bb487e077 100644
--- a/modules/node.module
+++ b/modules/node.module
@@ -213,7 +213,7 @@ function node_teaser($body) {
   ** Nevermind, we split it the hard way ...
   */
 
-  return substr($body, 0, $size);
+  return truncate_utf8($body, $size);
 }
 
 
diff --git a/modules/node/node.module b/modules/node/node.module
index c9392f776..bb487e077 100644
--- a/modules/node/node.module
+++ b/modules/node/node.module
@@ -213,7 +213,7 @@ function node_teaser($body) {
   ** Nevermind, we split it the hard way ...
   */
 
-  return substr($body, 0, $size);
+  return truncate_utf8($body, $size);
 }
 
 
diff --git a/modules/watchdog.module b/modules/watchdog.module
index a7ed775a8..e0b1c38d2 100644
--- a/modules/watchdog.module
+++ b/modules/watchdog.module
@@ -94,7 +94,7 @@ function watchdog_overview($type) {
   while ($watchdog = db_fetch_object($result)) {
     $rows[] = array(
       array("data" => format_date($watchdog->timestamp, "small"), "class" => "watchdog-$watchdog->type"),
-      array("data" => substr(strip_tags($watchdog->message), 0, 64), "class" => "watchdog-$watchdog->type"),
+      array("data" => truncate_utf8(strip_tags($watchdog->message), 64), "class" => "watchdog-$watchdog->type"),
       array("data" => format_name($watchdog), "class" => "watchdog-$watchdog->type"),
       array("data" => $watchdog->link, "class" => "watchdog-$watchdog->type"),
       array("data" => l(t("view details"), "admin/watchdog/view/$watchdog->wid"), "class" => "watchdog-$watchdog->type")
diff --git a/modules/watchdog/watchdog.module b/modules/watchdog/watchdog.module
index a7ed775a8..e0b1c38d2 100644
--- a/modules/watchdog/watchdog.module
+++ b/modules/watchdog/watchdog.module
@@ -94,7 +94,7 @@ function watchdog_overview($type) {
   while ($watchdog = db_fetch_object($result)) {
     $rows[] = array(
       array("data" => format_date($watchdog->timestamp, "small"), "class" => "watchdog-$watchdog->type"),
-      array("data" => substr(strip_tags($watchdog->message), 0, 64), "class" => "watchdog-$watchdog->type"),
+      array("data" => truncate_utf8(strip_tags($watchdog->message), 64), "class" => "watchdog-$watchdog->type"),
       array("data" => format_name($watchdog), "class" => "watchdog-$watchdog->type"),
       array("data" => $watchdog->link, "class" => "watchdog-$watchdog->type"),
       array("data" => l(t("view details"), "admin/watchdog/view/$watchdog->wid"), "class" => "watchdog-$watchdog->type")