- Patch #19298 by Jeremy: loose caching!

Drupal's existing caching mechanism doesn't perform well on highly dynamic websites in which the cache is flushed frequently. One example is a site that is under attack by a spambot that is posting spam comments every few seconds, causing all cached pages to be flushed every few seconds.  Loose caching immediately flushes the cache only for specific users who have modified cached data (whether or not they are logged in), delaying the flushing of data for other users by several minutes.

(I rewrote the help text a bit and made minor changes to the code comments.)

9 files changed, 84 insertions, 15 deletions

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 283710ea3..a72f3d40e 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -2,6 +2,8 @@ Drupal x.x.x, xxxx-xx-xx
 ------------------------
 
 - added free tagging support (folksonomies).
+- performance:
+    * added 'loose caching' option for high-traffic sites.
 
 Drupal x.x.x, xxxx-xx-xx
 ------------------------
diff --git a/database/database.mysql b/database/database.mysql
index b50d80cab..fbb2e76b5 100644
--- a/database/database.mysql
+++ b/database/database.mysql
@@ -574,6 +574,7 @@ CREATE TABLE sessions (
   sid varchar(32) NOT NULL default '',
   hostname varchar(128) NOT NULL default '',
   timestamp int(11) NOT NULL default '0',
+  cache int(11) NOT NULL default '0',
   session longtext,
   KEY uid (uid),
   PRIMARY KEY (sid),
diff --git a/database/database.pgsql b/database/database.pgsql
index 64a63215d..a911049f0 100644
--- a/database/database.pgsql
+++ b/database/database.pgsql
@@ -587,6 +587,7 @@ CREATE TABLE sessions (
   sid varchar(32) NOT NULL default '',
   hostname varchar(128) NOT NULL default '',
   timestamp integer NOT NULL default '0',
+  cache integer NOT NULL default '0',
   session text,
   PRIMARY KEY (sid)
 );
diff --git a/database/updates.inc b/database/updates.inc
index 076d647ad..19eac2c26 100644
--- a/database/updates.inc
+++ b/database/updates.inc
@@ -105,7 +105,8 @@ $sql_updates = array(
   "2005-03-03" => "update_126",
   "2005-03-18" => "update_127",
   "2005-03-21" => "update_128",
-  "2005-04-08: first update since Drupal 4.6.0 release" => "update_129"
+  "2005-04-08: first update since Drupal 4.6.0 release" => "update_129",
+  "2005-04-10" => "update_130"
 );
 
 function update_32() {
@@ -2364,4 +2365,15 @@ function update_129() {
   return $ret;
 }
 
+function update_130() {
+  $ret = array();
+  if ($GLOBALS['db_type'] == 'mysql') {
+    $ret[] = update_sql("ALTER TABLE sessions ADD cache int(11) NOT NULL default '0' AFTER timestamp");
+  }
+  elseif ($GLOBALS['db_type'] == 'pgsql') {
+    $ret[] = update_sql("ALTER TABLE sessions ADD cache int(11) NOT NULL default '0' AFTER timestamp");
+  }
+  return $ret;
+}
+
 ?>
diff --git a/includes/bootstrap.inc b/includes/bootstrap.inc
index 7ad4626a3..def6e412c 100644
--- a/includes/bootstrap.inc
+++ b/includes/bootstrap.inc
@@ -9,6 +9,10 @@
 define('CACHE_PERMANENT', 0);
 define('CACHE_TEMPORARY', -1);
 
+define('CACHE_DISABLED', 0);
+define('CACHE_ENABLED_STRICT', 1);
+define('CACHE_ENABLED_LOOSE', 2);
+
 define('WATCHDOG_NOTICE', 0);
 define('WATCHDOG_WARNING', 1);
 define('WATCHDOG_ERROR', 2);
@@ -196,9 +200,36 @@ function variable_del($name) {
  *   The cache ID of the data to retrieve.
  */
 function cache_get($key) {
-  $cache = db_fetch_object(db_query("SELECT data, created, headers FROM {cache} WHERE cid = '%s'", $key));
+  global $user;
+  $sid = session_id();
+
+  // CACHE_ENABLED_LOOSE garbage collection
+  $cache_flush = variable_get('cache_flush', 0);
+  if ($cache_flush && ($cache_flush + variable_get('cache_flush_delay', 300) <= time())) {
+    // Time to flush old cache data
+    db_query("DELETE FROM {cache} WHERE expire != %d AND expire <= %d", CACHE_PERMANENT, $cache_flush);
+    variable_set('cache_flush', 0);
+  }
+
+  $cache = db_fetch_object(db_query("SELECT data, created, headers, expire FROM {cache} WHERE cid = '%s'", $key));
   if (isset($cache->data)) {
-    $cache->data = db_decode_blob($cache->data);
+    // If data is permanent or using strict caching, always return data.
+    if ($cache->expire == CACHE_PERMANENT || variable_get('cache', CACHE_DISABLED) == CACHE_ENABLED_STRICT) {
+      $cache->data = db_decode_blob($cache->data);
+    }
+    // If using loose caching, validate data is current before we return it by
+    // making sure the cache entry was created before the timestamp in the
+    // current session's cache timer.  The cache variable is already loaded
+    // into the $user object by sess_read in session.inc.
+    else {
+      if ($user->cache > $cache->created) {
+        // This cache data is too old and thus not valid for us, ignore it.
+        return 0;
+      }
+      else {
+        $cache->data = db_decode_blob($cache->data);
+      }
+    }
     return $cache;
   }
   return 0;
@@ -235,16 +266,41 @@ function cache_set($cid, $data, $expire = CACHE_PERMANENT, $headers = NULL) {
  * Expire data from the cache.
  *
  * @param $cid
- *   If set, the cache ID to delete. Otherwise, all cache entries that can expire
- *   are deleted.
+ *   If set, the cache ID to delete. Otherwise, all cache entries that can
+ *   expire are deleted.
  *
  * @param $wildcard
  *   If set to true, the $cid is treated as a substring to match rather than a
  *   complete ID.
  */
 function cache_clear_all($cid = NULL, $wildcard = false) {
+  global $user;
+  $sid = session_id();
+
   if (empty($cid)) {
-    db_query("DELETE FROM {cache} WHERE expire != %d AND expire < %d", CACHE_PERMANENT, time());
+    if (variable_get('cache', CACHE_DISABLED) == CACHE_ENABLED_STRICT) {
+      // Strict caching, flush all temporary cache entries:
+      db_query("DELETE FROM {cache} WHERE expire != %d AND expire < %d", CACHE_PERMANENT, time());
+    }
+    else {
+      $cache_flush = variable_get('cache_flush', 0);
+      // Loose caching, only flush temporary cache entries that have been
+      // invalidated for more than maximum allowable time.
+      if ($cache_flush && ($cache_flush + variable_get('cache_flush_delay', 300) <= time())) {
+        // Only flush cache data older than $cache_flush, as newer data may
+        // now be valid.
+        db_query("DELETE FROM {cache} WHERE expire != %d AND expire <= %d", CACHE_PERMANENT, $cache_flush);
+        variable_set('cache_flush', 0);
+      }
+      // Invalidate temporary cache data only for current user/session.  We
+      // set $user->cache, which gets saved into the sessions table by
+      // sess_write() in session.inc.
+      $user->cache = time();
+      if (variable_get('cache_flush', 0) == 0) {
+        // Set timestamp to know which cache entries we eventually clear:
+        variable_set('cache_flush', time());
+      }
+    }
   }
   else {
     if ($wildcard) {
diff --git a/includes/session.inc b/includes/session.inc
index 73514e37d..517651704 100644
--- a/includes/session.inc
+++ b/includes/session.inc
@@ -45,7 +45,7 @@ function sess_read($key) {
 function sess_write($key, $value) {
   global $user;
 
-  db_query("UPDATE {sessions} SET uid = %d, hostname = '%s', session = '%s', timestamp = %d WHERE sid = '%s'", $user->uid, $_SERVER["REMOTE_ADDR"], $value, time(), $key);
+  db_query("UPDATE {sessions} SET uid = %d, cache = %d, hostname = '%s', session = '%s', timestamp = %d WHERE sid = '%s'", $user->uid, $user->cache, $_SERVER["REMOTE_ADDR"], $value, time(), $key);
 
   return '';
 }
diff --git a/modules/system.module b/modules/system.module
index 5d75bba29..a5d4133a4 100644
--- a/modules/system.module
+++ b/modules/system.module
@@ -35,9 +35,7 @@ function system_help($section) {
       <p>If your hosting company does not allow you to set up crontab entries, you can always ask someone else to set up an entry for you. After all, virtually any Unix/Linux machine with access to the internet can set up a crontab entry to frequently visit %cron-link.</p>
       <p>For the Unix/Linux crontab itself, use a browser like <a href=\"%lynx\">lynx</a> or <a href=\"%wget\">wget</a> but make sure the process terminates: either use <code>/usr/bin/lynx -source %base_url/cron.php</code> or <code>/usr/bin/wget -o /dev/null -O /dev/null %cron-link</code>.  Take a look at the example scripts in the <code>scripts</code>-directory. Make sure to adjust them to fit your needs.  A good crontab line to run the cron script once every hour would be:
       <pre>     00 * * * * /home/www/drupal/scripts/cron-lynx.sh</pre>
-      Note that it is essential to access <code>cron.php</code> using a browser on the web site's domain; do not run it using command line PHP and avoid using <code>localhost</code> or <code>127.0.0.1</code> or some of the environment variables will not be set correctly and features may not work as expected.</p>
-      <h3><a id=\"cache\">Cache</a></h3>
-      <p>Drupal has a caching mechanism which stores dynamically generated web pages in a database.  By caching a web page, Drupal does not have to create the page each time someone wants to view it, instead it takes only one SQL query to display it, reducing response time and the server's load.  Only pages requested by \"anonymous\" users are cached.  In order to reduce server load and save bandwidth, Drupal stores and sends cached pages compressed.</p>", array('%base_url' => $base_url, '%cron-link' => "<a href=\"$base_url/cron.php\">$base_url/cron.php</a>", '%lynx' => 'http://lynx.browser.org', '%wget' => 'http://www.gnu.org/software/wget/wget.html' ));
+      Note that it is essential to access <code>cron.php</code> using a browser on the web site's domain; do not run it using command line PHP and avoid using <code>localhost</code> or <code>127.0.0.1</code> or some of the environment variables will not be set correctly and features may not work as expected.</p>", array('%base_url' => $base_url, '%cron-link' => "<a href=\"$base_url/cron.php\">$base_url/cron.php</a>", '%lynx' => 'http://lynx.browser.org', '%wget' => 'http://www.gnu.org/software/wget/wget.html' ));
     case 'admin/modules#description':
       return t('Handles general site configuration for administrators.');
   }
@@ -225,7 +223,7 @@ function system_view_general() {
   $output .= form_group(t('Error handling'), $group);
 
   // Caching:
-  $group  = form_radios(t('Cache support'), 'cache', variable_get('cache', 0), array(t('Disabled'), t('Enabled')), t('Enable or disable the caching of rendered pages.  When caching is enabled, Drupal will flush the cache when required to make sure updates take effect immediately.  Check the <a href="%documentation">cache documentation</a> for information on Drupal\'s cache system.', array('%documentation' => url('admin/help', NULL, NULL, 'cache'))));
+  $group  = form_radios(t('Page cache'), 'cache', variable_get('cache', CACHE_DISABLED), array(CACHE_DISABLED => t('Disabled (low-traffic sites)'), CACHE_ENABLED_STRICT => t('Strict (medium-traffic sites)'), CACHE_ENABLED_LOOSE => t('Loose (high-traffic sites)')), t("Drupal has a caching mechanism which stores dynamically generated web pages in a database.  By caching a web page, Drupal does not have to create the page each time someone wants to view it, instead it takes only one SQL query to display it, reducing response time and the server's load.  Only pages requested by \"anonymous\" users are cached.  In order to reduce server load and save bandwidth, Drupal stores and sends cached pages compressed.  Drupal supports strict caching and loose caching.  Strict caching immediately deletes cached data as soon as it becomes invalid for any user.  Loose caching delays the deletion of cached data to provide better performance for high traffic sites."));
 
   $output .= form_group(t('Cache settings'), $group);
 
diff --git a/modules/system/system.module b/modules/system/system.module
index 5d75bba29..a5d4133a4 100644
--- a/modules/system/system.module
+++ b/modules/system/system.module
@@ -35,9 +35,7 @@ function system_help($section) {
       <p>If your hosting company does not allow you to set up crontab entries, you can always ask someone else to set up an entry for you. After all, virtually any Unix/Linux machine with access to the internet can set up a crontab entry to frequently visit %cron-link.</p>
       <p>For the Unix/Linux crontab itself, use a browser like <a href=\"%lynx\">lynx</a> or <a href=\"%wget\">wget</a> but make sure the process terminates: either use <code>/usr/bin/lynx -source %base_url/cron.php</code> or <code>/usr/bin/wget -o /dev/null -O /dev/null %cron-link</code>.  Take a look at the example scripts in the <code>scripts</code>-directory. Make sure to adjust them to fit your needs.  A good crontab line to run the cron script once every hour would be:
       <pre>     00 * * * * /home/www/drupal/scripts/cron-lynx.sh</pre>
-      Note that it is essential to access <code>cron.php</code> using a browser on the web site's domain; do not run it using command line PHP and avoid using <code>localhost</code> or <code>127.0.0.1</code> or some of the environment variables will not be set correctly and features may not work as expected.</p>
-      <h3><a id=\"cache\">Cache</a></h3>
-      <p>Drupal has a caching mechanism which stores dynamically generated web pages in a database.  By caching a web page, Drupal does not have to create the page each time someone wants to view it, instead it takes only one SQL query to display it, reducing response time and the server's load.  Only pages requested by \"anonymous\" users are cached.  In order to reduce server load and save bandwidth, Drupal stores and sends cached pages compressed.</p>", array('%base_url' => $base_url, '%cron-link' => "<a href=\"$base_url/cron.php\">$base_url/cron.php</a>", '%lynx' => 'http://lynx.browser.org', '%wget' => 'http://www.gnu.org/software/wget/wget.html' ));
+      Note that it is essential to access <code>cron.php</code> using a browser on the web site's domain; do not run it using command line PHP and avoid using <code>localhost</code> or <code>127.0.0.1</code> or some of the environment variables will not be set correctly and features may not work as expected.</p>", array('%base_url' => $base_url, '%cron-link' => "<a href=\"$base_url/cron.php\">$base_url/cron.php</a>", '%lynx' => 'http://lynx.browser.org', '%wget' => 'http://www.gnu.org/software/wget/wget.html' ));
     case 'admin/modules#description':
       return t('Handles general site configuration for administrators.');
   }
@@ -225,7 +223,7 @@ function system_view_general() {
   $output .= form_group(t('Error handling'), $group);
 
   // Caching:
-  $group  = form_radios(t('Cache support'), 'cache', variable_get('cache', 0), array(t('Disabled'), t('Enabled')), t('Enable or disable the caching of rendered pages.  When caching is enabled, Drupal will flush the cache when required to make sure updates take effect immediately.  Check the <a href="%documentation">cache documentation</a> for information on Drupal\'s cache system.', array('%documentation' => url('admin/help', NULL, NULL, 'cache'))));
+  $group  = form_radios(t('Page cache'), 'cache', variable_get('cache', CACHE_DISABLED), array(CACHE_DISABLED => t('Disabled (low-traffic sites)'), CACHE_ENABLED_STRICT => t('Strict (medium-traffic sites)'), CACHE_ENABLED_LOOSE => t('Loose (high-traffic sites)')), t("Drupal has a caching mechanism which stores dynamically generated web pages in a database.  By caching a web page, Drupal does not have to create the page each time someone wants to view it, instead it takes only one SQL query to display it, reducing response time and the server's load.  Only pages requested by \"anonymous\" users are cached.  In order to reduce server load and save bandwidth, Drupal stores and sends cached pages compressed.  Drupal supports strict caching and loose caching.  Strict caching immediately deletes cached data as soon as it becomes invalid for any user.  Loose caching delays the deletion of cached data to provide better performance for high traffic sites."));
 
   $output .= form_group(t('Cache settings'), $group);
 
diff --git a/sites/default/settings.php b/sites/default/settings.php
index 50499134e..532a37143 100644
--- a/sites/default/settings.php
+++ b/sites/default/settings.php
@@ -88,6 +88,7 @@ $db_prefix = '';
  * a trailing slash; Drupal will add it for you.
  */
 $base_url = 'http://localhost';
+$base_url = 'http://169.254.115.224/~dries/drupal-cvs';
 
 /**
  * PHP settings: