- Patch #251792 by pwolanin, Damien Tournoud, slantview, c960657: added a locking framework for long running operations.

8 files changed, 403 insertions, 3 deletions

diff --git a/includes/bootstrap.inc b/includes/bootstrap.inc
index 5abccfeb0..97fff169a 100644
--- a/includes/bootstrap.inc
+++ b/includes/bootstrap.inc
@@ -1513,6 +1513,9 @@ function _drupal_bootstrap($phase) {
       }
 
       // Prepare for non-cached page workflow.
+      require_once DRUPAL_ROOT . '/' . variable_get('lock_inc', 'includes/lock.inc');
+      lock_initialize();
+
       if ($_SERVER['SERVER_SOFTWARE'] !== 'PHP CLI') {
         ob_start();
         drupal_page_header();
diff --git a/includes/lock.inc b/includes/lock.inc
new file mode 100644
index 000000000..5d03cdd99
--- /dev/null
+++ b/includes/lock.inc
@@ -0,0 +1,252 @@
+<?php
+// $Id$
+
+/**
+ * @file
+ * A database-mediated implementation of a locking mechanism.
+ */
+
+/**
+ * @defgroup lock Functions to coordinate long-running operations across requests.
+ * @{
+ * In most environments, multiple Drupal page requests (a.k.a. threads or
+ * processes) will execute in parallel. This leads to potential conflicts or
+ * race conditions when two requests execute the same code at the same time. A
+ * common example of this is a rebuild like menu_rebuild() where we invoke many
+ * hook implementations to get and process data from all active modules, and
+ * then delete the current data in the database to insert the new afterwards.
+ *
+ * This is a cooperative, advisory lock system. Any long-running operation
+ * that could potentially be attempted in parallel by multiple requests should
+ * try to acquire a lock before proceeding. By obtainng a lock, one request
+ * notifies any other requests that a specific operation is in progress which
+ * must not be executed in parallel.
+ *
+ * To use this API, pick a unique name for the lock. A sensible choice is the
+ * name of the function performing the operation. A very simple example use of
+ * this API:
+ * @code
+ * function mymodule_long_operation() {
+ *   if (lock_acquire('mymodule_long_operation')) {
+ *     // Do the long operation here.
+ *     // ...
+ *     lock_release('mymodule_long_operation');
+ *   }
+ * }
+ * @endcode
+ *
+ * If a function acquires a lock it should always release it when the
+ * operation is complete by calling lock_release(), as in the example.
+ *
+ * A function that has acquired a lock may attempt to renew a lock (extend the
+ * duration of the lock) by calling lock_acquire() again during the operation.
+ * Failure to renew a lock is indicative that another request has acquired
+ * the lock, and that the current operation may need to be aborted.
+ *
+ * If a function fails to acquire a lock it may either immediately return, or
+ * it may call lock_wait() if the rest of the current page request requires
+ * that the operation in question be complete. After lock_wait() returns,
+ * the function may again attempt to acquire the lock, or may simply allow the
+ * page request to proceed on the assumption that a parallel request completed
+ * the operation.
+ *
+ * lock_acquire() and lock_wait() will automatically break (delete) a lock
+ * whose duration has exceeded the timeout specified when it was acquired.
+ *
+ * A function that has acquired a lock may attempt to renew a lock (extend the
+ * duration of the lock) by calling lock_acquire() again during the operation.
+ * Failure to renew a lock is indicative that another request has acquired
+ * the lock, and that the current operation may need to be aborted.
+ *
+ * Alternative implementations of this API (such as APC) may be substituted
+ * by setting the 'lock_inc' variable to an alternate include filepath. Since
+ * this is an API intended to support alternative implementations, code using
+ * this API should never rely upon specific implementation details (for example
+ * no code should look for or directly modify a lock in the {semaphore} table).
+ */
+
+/**
+ * Initialize the locking system.
+ */
+function lock_initialize() {
+  global $locks;
+
+  $locks = array();
+}
+
+/**
+ * Helper function to get this request's unique id.
+ */
+function _lock_id() {
+  $lock_id = &drupal_static(__FUNCTION__);
+
+  if (!isset($lock_id)) {
+    // Assign a unique id.
+    $lock_id = uniqid(mt_rand(), TRUE);
+    // We only register a shutdown function if a lock is used.
+    register_shutdown_function('lock_release_all', $lock_id);
+  }
+  return $lock_id;
+}
+
+/**
+ * Acquire (or renew) a lock, but do not block if it fails.
+ *
+ * @param $name
+ *   The name of the lock.
+ * @param $timeout
+ *   A number of seconds (float) before the lock expires (minimum of 0.001).
+ * @return
+ *   TRUE if the lock was acquired, FALSE if it failed.
+ */
+function lock_acquire($name, $timeout = 30.0) {
+  global $locks;
+
+  // Insure that the timeout is at least 1 ms.
+  $timeout = max($timeout, 0.001);
+  $expire = microtime(TRUE) + $timeout;
+  if (isset($locks[$name])) {
+    // Try to extend the expiration of a lock we already acquired.
+    $success = (bool) db_update('semaphore')
+      ->fields(array('expire' => $expire))
+      ->condition('name', $name)
+      ->condition('value', _lock_id())
+      ->execute();
+    if (!$success) {
+      // The lock was broken.
+      unset($locks[$name]);
+    }
+    return $success;
+  }
+  else {
+    // Optimistically try to acquire the lock, then retry once if it fails.
+    // The first time through the loop cannot be a retry.
+    $retry = FALSE;
+    // We always want to do this code at least once.
+    do {
+      try {
+        db_insert('semaphore')
+          ->fields(array(
+            'name' => $name,
+            'value' => _lock_id(),
+            'expire' => $expire,
+          ))
+          ->execute();
+        // We track all acquired locks in the global variable.
+        $locks[$name] = TRUE;
+        // We never need to try again.
+        $retry = FALSE;
+      }
+      catch (PDOException $e) {
+        // Suppress the error. If this is our first pass through the loop,
+        // then $retry is FALSE. In this case, the insert must have failed
+        // meaning some other request acquired the lock but did not release it.
+        // We decide whether to retry by checking lock_may_be_available()
+        // Since this will break the lock in case it is expired.
+        $retry = $retry ? FALSE : lock_may_be_available($name);
+      }
+      // We only retry in case the first attempt failed, but we then broke
+      // an expired lock.
+    } while ($retry);
+  }
+  return isset($locks[$name]);
+}
+
+/**
+ * Check if lock acquired by a different process may be available.
+ *
+ * If an existing lock has expired, it is removed.
+ *
+ * @param $name
+ *   The name of the lock.
+ * @return
+ *   TRUE if there is no lock or it was removed, FALSE otherwise.
+ */
+function lock_may_be_available($name) {
+  $lock = db_query('SELECT expire, value FROM {semaphore} WHERE name = :name', array(':name' => $name))->fetchAssoc();
+  if (!$lock) {
+    return TRUE;
+  }
+  $expire = (float) $lock['expire'];
+  $now = microtime(TRUE);
+  if ($now > $expire) {
+    // We check two conditions to prevent a race condition where another
+    // request acquired the lock and set a new expire time. We add a small
+    // number to $expire to avoid errors with float to string conversion.
+    return (bool) db_delete('semaphore')
+      ->condition('name', $name)
+      ->condition('value', $lock['value'])
+      ->condition('expire', 0.0001 + $expire, '<=')
+      ->execute();
+  }
+  return FALSE;
+}
+
+/**
+ * Wait for a lock to be available.
+ *
+ * This function may be called in a request that fails to acquire a desired
+ * lock. This will block further execution until the lock is available or the
+ * specified delay in seconds is reached. This should not be used with locks
+ * that are acquired very frequently, since the lock is likely to be acquired
+ * again by a different request during the sleep().
+ *
+ * @param $name
+ *   The name of the lock.
+ * @param $delay
+ *   The maximum number of seconds to wait, as an integer.
+ * @return
+ *   TRUE if the lock holds, FALSE if it is available.
+ */
+function lock_wait($name, $delay = 30) {
+  $delay = (int) $delay;
+  while ($delay--) {
+    // This function should only be called by a request that failed to get a
+    // lock, so we sleep first to give the parallel request a chance to finish
+    // and release the lock.
+    sleep(1);
+    if (lock_may_be_available($name)) {
+      // No longer need to wait.
+      return FALSE;
+    }
+  }
+  // The caller must still wait longer to get the lock.
+  return TRUE;
+}
+
+/**
+ * Release a lock previously acquired by lock_acquire().
+ *
+ * This will release the named lock if it is still held by the current request.
+ *
+ * @param $name
+ *   The name of the lock.
+ */
+function lock_release($name) {
+  global $locks;
+
+  unset($locks[$name]);
+  db_delete('semaphore')
+    ->condition('name', $name)
+    ->condition('value', _lock_id())
+    ->execute();
+}
+
+/**
+ * Release all previously acquired locks.
+ */
+function lock_release_all($lock_id = NULL) {
+  global $locks;
+
+  $locks = array();
+  if (empty($lock_id)) {
+    $lock_id = _lock_id();
+  }
+  db_delete('semaphore')
+    ->condition('value', $lock_id)
+    ->execute();
+}
+
+/**
+ * @} End of "defgroup lock".
+ */
diff --git a/includes/menu.inc b/includes/menu.inc
index 4ad0770be..de37139a0 100644
--- a/includes/menu.inc
+++ b/includes/menu.inc
@@ -1835,18 +1835,35 @@ function menu_cache_clear_all() {
  * schedule a call to itself on the first real page load from
  * menu_execute_active_handler(), because the maintenance page environment
  * is different and leaves stale data in the menu tables.
+ *
+ * @return
+ *   TRUE if the menu was rebuilt, FALSE if another thread was rebuilding
+ *   in parallel and the current thread just waited for completion.
  */
 function menu_rebuild() {
-  variable_del('menu_rebuild_needed');
+  if (!lock_acquire('menu_rebuild')) {
+    // Wait for another request that is already doing this work.
+    // We choose to block here since otherwise the router item may not 
+    // be available in menu_execute_active_handler() resulting in a 404.
+    lock_wait('menu_rebuild');
+    return FALSE;
+  }
+
   list($menu, $masks) = menu_router_build();
   _menu_router_save($menu, $masks);
   _menu_navigation_links_rebuild($menu);
+  // Clear the menu, page and block caches.
   menu_cache_clear_all();
-  // Clear the page and block caches.
   _menu_clear_page_cache();
+
   if (defined('MAINTENANCE_MODE')) {
     variable_set('menu_rebuild_needed', TRUE);
   }
+  else {
+    variable_del('menu_rebuild_needed');
+  }
+  lock_release('menu_rebuild');
+  return TRUE;
 }
 
 /**
diff --git a/modules/locale/locale.module b/modules/locale/locale.module
index cb37d8cbf..dcb046cdf 100644
--- a/modules/locale/locale.module
+++ b/modules/locale/locale.module
@@ -375,7 +375,7 @@ function locale($string = NULL, $context = NULL, $langcode = NULL, $reset = FALS
       if ($cache = cache_get('locale:' . $langcode, 'cache')) {
         $locale_t[$langcode] = $cache->data;
       }
-      else {
+      elseif (lock_acquire('locale_cache_' . $langcode)) {
         // Refresh database stored cache of translations for given language.
         // We only store short strings used in current version, to improve
         // performance and consume less memory.
@@ -384,6 +384,7 @@ function locale($string = NULL, $context = NULL, $langcode = NULL, $reset = FALS
           $locale_t[$langcode][$data->context][$data->source] = (empty($data->translation) ? TRUE : $data->translation);
         }
         cache_set('locale:' . $langcode, $locale_t[$langcode]);
+        lock_release('locale_cache_' . $langcode);
       }
     }
   }
diff --git a/modules/simpletest/simpletest.info b/modules/simpletest/simpletest.info
index dc3974902..9d307eb5b 100644
--- a/modules/simpletest/simpletest.info
+++ b/modules/simpletest/simpletest.info
@@ -24,6 +24,7 @@ files[] = tests/filetransfer.test
 files[] = tests/form.test
 files[] = tests/graph.test
 files[] = tests/image.test
+files[] = tests/lock.test
 files[] = tests/menu.test
 files[] = tests/module.test
 files[] = tests/registry.test
diff --git a/modules/simpletest/tests/lock.test b/modules/simpletest/tests/lock.test
new file mode 100644
index 000000000..d9fc53f75
--- /dev/null
+++ b/modules/simpletest/tests/lock.test
@@ -0,0 +1,58 @@
+<?php
+// $Id$
+
+/**
+ * Tests for the lock system.
+ */
+class LockFunctionalTest extends DrupalWebTestCase {
+
+  public static function getInfo() {
+    return array(
+      'name' => 'Locking framework tests',
+      'description' => 'Confirm locking works between two separate requests.',
+      'group' => 'System',
+    );
+  }
+
+  function setUp() {
+    parent::setUp('system_test');
+  }
+
+  /**
+   * Confirm that we can acquire and release locks in two parallel requests.
+   */
+  function testLockAcquire() {
+    $lock_acquired = 'TRUE: Lock successfully acquired in system_test_lock_acquire()';
+    $lock_not_acquired = 'FALSE: Lock not acquired in system_test_lock_acquire()';
+    $this->assertTrue(lock_acquire('system_test_lock_acquire'), t('Lock acquired by this request.'), t('Lock'));
+    $this->assertTrue(lock_acquire('system_test_lock_acquire'), t('Lock extended by this request.'), t('Lock'));
+    lock_release('system_test_lock_acquire');
+
+    // Cause another request to acquire the lock.
+    $this->drupalGet('system-test/lock-acquire');
+    $this->assertText($lock_acquired, t('Lock acquired by the other request.'), t('Lock'));
+    // The other request has finished, thus it should have released its lock.
+    $this->assertTrue(lock_acquire('system_test_lock_acquire'), t('Lock acquired by this request.'), t('Lock'));
+    // This request holds the lock, so the other request cannot acquire it.
+    $this->drupalGet('system-test/lock-acquire');
+    $this->assertText($lock_not_acquired, t('Lock not acquired by the other request.'), t('Lock'));
+    lock_release('system_test_lock_acquire');
+
+    // Try a very short timeout and lock breaking.
+    $this->assertTrue(lock_acquire('system_test_lock_acquire', 0.5), t('Lock acquired by this request.'), t('Lock'));
+    sleep(1);
+    // The other request should break our lock.
+    $this->drupalGet('system-test/lock-acquire');
+    $this->assertText($lock_acquired, t('Lock acquired by the other request, breaking our lock.'), t('Lock'));
+    // We cannot renew it, since the other thread took it.
+    $this->assertFalse(lock_acquire('system_test_lock_acquire'), t('Lock cannot be extended by this request.'), t('Lock'));
+
+    // Check the shut-down function.
+    $lock_acquired_exit = 'TRUE: Lock successfully acquired in system_test_lock_exit()';
+    $lock_not_acquired_exit = 'FALSE: Lock not acquired in system_test_lock_exit()';
+    $this->drupalGet('system-test/lock-exit');
+    $this->assertText($lock_acquired_exit, t('Lock acquired by the other request before exit.'), t('Lock'));
+    $this->assertTrue(lock_acquire('system_test_lock_exit'), t('Lock acquired by this request after the other request exits.'), t('Lock'));
+  }
+}
+
diff --git a/modules/simpletest/tests/system_test.module b/modules/simpletest/tests/system_test.module
index 3f0a1acef..90a8f28a0 100644
--- a/modules/simpletest/tests/system_test.module
+++ b/modules/simpletest/tests/system_test.module
@@ -59,6 +59,20 @@ function system_test_menu() {
     'type' => MENU_CALLBACK,
   );
 
+  $items['system-test/lock-acquire'] = array(
+    'title' => 'Lock acquire',
+    'page callback' => 'system_test_lock_acquire',
+    'access callback' => TRUE,
+    'type' => MENU_CALLBACK,
+  );
+
+  $items['system-test/lock-exit'] = array(
+    'title' => 'Lock acquire then exit',
+    'page callback' => 'system_test_lock_exit',
+    'access callback' => TRUE,
+    'type' => MENU_CALLBACK,
+  );
+
   return $items;
 }
 
@@ -185,3 +199,29 @@ function system_test_system_info_alter(&$info, $file) {
   }
 }
 
+/**
+ * Try to acquire a named lock and report the outcome.
+ */
+function system_test_lock_acquire() {
+  if (lock_acquire('system_test_lock_acquire')) {
+    lock_release('system_test_lock_acquire');
+    return 'TRUE: Lock successfully acquired in system_test_lock_acquire()';
+  }
+  else {
+    return 'FALSE: Lock not acquired in system_test_lock_acquire()';
+  }
+}
+
+/**
+ * Try to acquire a specific lock, and then exit.
+ */
+function system_test_lock_exit() {
+  if (lock_acquire('system_test_lock_exit', 900)) {
+    echo 'TRUE: Lock successfully acquired in system_test_lock_exit()';
+    // The shut-down function should release the lock.
+    exit();
+  }
+  else {
+    return 'FALSE: Lock not acquired in system_test_lock_exit()';
+  }
+}
diff --git a/modules/system/system.install b/modules/system/system.install
index 3ff1c5905..83a0f268b 100644
--- a/modules/system/system.install
+++ b/modules/system/system.install
@@ -1265,6 +1265,34 @@ function system_schema() {
     'primary key' => array('filename'),
   );
 
+  $schema['semaphore'] = array(
+    'description' => 'Table for holding semaphores, locks, flags, etc. that cannot be stored as Drupal variables since they must not be cached.',
+    'fields' => array(
+      'name' => array(
+        'description' => 'Primary Key: Unique name.',
+        'type' => 'varchar',
+        'length' => 255,
+        'not null' => TRUE,
+        'default' => ''
+      ),
+      'value' => array(
+        'description' => 'A value for the semaphore.',
+        'type' => 'varchar',
+        'length' => 255,
+        'not null' => TRUE,
+        'default' => ''
+      ),
+      'expire' => array(
+        'description' => 'A Unix timestamp with microseconds indicating when the semaphore should expire.',
+        'type' => 'float',
+        'size' => 'big',
+        'not null' => TRUE
+      ),
+    ),
+    'indexes' => array('value' => array('value')),
+    'primary key' => array('name'),
+  );
+
   $schema['sessions'] = array(
     'description' => "Drupal's session handlers read and write into the sessions table. Each record represents a user session, either anonymous or authenticated.",
     'fields' => array(