#895176 by ksenzee, dereine, mongolito404, Berdir, moshe weitzman, chx: Fixed upgrade fails because of integrity constraint violation in system_update_7060.

1 files changed, 165 insertions, 125 deletions

diff --git a/modules/system/system.install b/modules/system/system.install
index 4caeaa94e..97e058e86 100644
--- a/modules/system/system.install
+++ b/modules/system/system.install
@@ -2586,132 +2586,169 @@ function system_update_7059() {
 }
 
 /**
- * Migrate upload.module to file.module.
+ * Create fields in preparation for migrating upload.module to file.module.
  */
-function system_update_7060(&$sandbox) {
+function system_update_7060() {
   if (!db_table_exists('upload')) {
     return;
   }
 
-  if (!isset($sandbox['progress'])) {
-    // Initialize batch update information.
-    $sandbox['progress'] = 0;
-    $sandbox['last_vid_processed'] = -1;
-    $sandbox['max'] = db_query("SELECT COUNT(DISTINCT u.vid) FROM {upload} u")->fetchField();
-
-    // Check which node types have upload.module attachments enabled.
-    $context['types'] = array();
-    foreach (node_type_get_types() as $node_type => $node_info) {
-      if (variable_get('upload_' . $node_type, 1)) {
-        $context['types'][$node_type] = $node_type;
-      }
-      variable_del('upload_' . $node_type);
+  // Check which node types have upload.module attachments enabled.
+  $context['types'] = array();
+  foreach (_update_7000_node_get_types() as $node_type) {
+    if (variable_get('upload_' . $node_type->type, 0)) {
+      $context['types'][$node_type->type] = $node_type->type;
     }
+  }
 
-    // The {upload} table will be deleted when this update is complete so we
-    // want to be careful to migrate all the data, even for node types that
-    // may have had attachments disabled after files were uploaded. Look for
-    // any other node types referenced by the upload records and add those to
-    // the list. The admin can always remove the field later.
-    $results = db_query('SELECT DISTINCT type FROM {node} n INNER JOIN {upload} u ON n.vid = u.vid');
-    foreach ($results as $row) {
-      if (!isset($context['types'][$row->type])) {
-        drupal_set_message(t('The content type %rowtype had uploads disabled but contained uploaded file data. Uploads have been re-enabled to migrate the existing data. You may delete the "File attachments" field in the %rowtype type if this data is not necessary.', array('%rowtype' => $row->type)));
-        $context['types'][$row->type] = $row->type;
-      }
+  // The {upload} table will be deleted when this update is complete so we
+  // want to be careful to migrate all the data, even for node types that
+  // may have had attachments disabled after files were uploaded. Look for
+  // any other node types referenced by the upload records and add those to
+  // the list. The admin can always remove the field later.
+  $results = db_query('SELECT DISTINCT type FROM {node} n INNER JOIN {node_revision} nr ON n.nid = nr.nid INNER JOIN {upload} u ON nr.vid = u.vid');
+  foreach ($results as $row) {
+    if (!isset($context['types'][$row->type])) {
+      drupal_set_message(t('The content type %rowtype had uploads disabled but contained uploaded file data. Uploads have been re-enabled to migrate the existing data. You may delete the "File attachments" field in the %rowtype type if this data is not necessary.', array('%rowtype' => $row->type)));
+      $context['types'][$row->type] = $row->type;
     }
+  }
+
+  // Create a single "upload" field on all the content types that have uploads
+  // enabled, then add an instance to each enabled type.
+  if (count($context['types']) > 0) {
+    module_enable(array('file'));
+    $field = array(
+      'field_name' => 'upload',
+      'type' => 'file',
+      'module' => 'file',
+      'locked' => FALSE,
+      'cardinality' => FIELD_CARDINALITY_UNLIMITED,
+      'translatable' => FALSE,
+      'settings' => array(
+        'display_field' => 1,
+        'display_default' => variable_get('upload_list_default', 1),
+        'uri_scheme' => file_default_scheme(),
+        'default_file' => 0,
+      ),
+    );
 
-    // Create a single "field_upload" field on all the content types that have
-    // uploads enabled, then add an instance to each enabled type.
-    if (count($context['types']) > 0) {
-      module_enable(array('file'));
-      module_load_include('inc', 'field', 'field.crud');
-
-      $field = array(
-        'field_name' => 'file',
-        'type' => 'file',
-        'locked' => FALSE,
-        'cardinality' => FIELD_CARDINALITY_UNLIMITED,
-        'translatable' => FALSE,
+    $upload_size = variable_get('upload_uploadsize_default', 1);
+    $instance = array(
+      'field_name' => 'upload',
+      'entity_type' => 'node',
+      'bundle' => NULL,
+      'label' => 'File attachments',
+      'required' => 0,
+      'description' => '',
+      'widget' => array(
+        'weight' => '1',
         'settings' => array(
-          'display_field' => 1,
-          'display_default' => variable_get('upload_list_default', 1),
-          'uri_scheme' => file_default_scheme(),
-          'default_file' => 0,
+          'progress_indicator' => 'throbber',
         ),
-      );
-
-      $upload_size = variable_get('upload_uploadsize_default', 1);
-      $instance = array(
-        'field_name' => 'file',
-        'entity_type' => 'node',
-        'bundle' => NULL,
-        'label' => 'File attachments',
-        'widget_type' => 'file_generic',
-        'required' => 0,
-        'description' => '',
-        'widget' => array(
-          'weight' => '1',
-          'settings' => array(
-            'progress_indicator' => 'throbber',
-          ),
-          'type' => 'file_generic',
+        'type' => 'file_generic',
+      ),
+      'settings' => array(
+        'max_filesize' => $upload_size ? ($upload_size . ' MB') : '',
+        'file_extensions' => variable_get('upload_extensions_default', 'jpg jpeg gif png txt doc xls pdf ppt pps odt ods odp'),
+        'file_directory' => '',
+        'description_field' => 1,
+      ),
+      'display' => array(
+        'default' => array(
+          'label' => 'hidden',
+          'type' => 'file_table',
+          'settings' => array(),
+          'weight' => 0,
+          'module' => 'file',
         ),
-        'settings' => array(
-          'max_filesize' => $upload_size ? ($upload_size . ' MB') : '',
-          'file_extensions' => variable_get('upload_extensions_default', 'jpg jpeg gif png txt doc xls pdf ppt pps odt ods odp'),
-          'file_directory' => '',
-          'description_field' => 1,
+        'full' => array(
+          'label' => 'hidden',
+          'type' => 'file_table',
+          'settings' => array(),
+          'weight' => 0,
+          'module' => 'file',
         ),
-        'display' => array(
-          'full' => array(
-            'label' => 'hidden',
-            'type' => 'file_table',
-            'settings' => array(),
-            'weight' => 0,
-            'module' => 'file',
-          ),
-          'teaser' => array(
-            'label' => 'hidden',
-            'type' => 'hidden',
-            'settings' => array(),
-            'weight' => 0,
-            'module' => NULL,
-          ),
-          'rss' => array(
-            'label' => 'hidden',
-            'type' => 'file_table',
-            'settings' => array(),
-            'weight' => 0,
-            'module' => 'file',
-          ),
+        'teaser' => array(
+          'label' => 'hidden',
+          'type' => 'hidden',
+          'settings' => array(),
+          'weight' => 0,
+          'module' => NULL,
         ),
-      );
+        'rss' => array(
+          'label' => 'hidden',
+          'type' => 'file_table',
+          'settings' => array(),
+          'weight' => 0,
+          'module' => 'file',
+        ),
+      ),
+    );
 
-      // Create the field. Save the field id for the data insertion later on.
-      $field = field_create_field($field);
-      $sandbox['field_id'] = $field['id'];
+    // Create the field.
+    _update_7000_field_create_field($field);
 
-      // Create the instances.
-      foreach ($context['types'] as $bundle) {
-        $instance['bundle'] = $bundle;
-        field_create_instance($instance);
-      }
-    }
-    else {
-      // No uploads or content types with uploads enabled.
-      db_drop_table('upload');
-      // We're done: return without specifying a #progress.
-      return;
+    // Create the instances.
+    foreach ($context['types'] as $bundle) {
+      $instance['bundle'] = $bundle;
+      _update_7000_field_create_instance($field, $instance);
+      // Now that the instance is created, we can safely delete any legacy
+      // node type information.
+      variable_del('upload_' . $bundle);
     }
   }
+  else {
+    // No uploads or content types with uploads enabled.
+    db_drop_table('upload');
+  }
+}
 
-  // Migrate a batch of files from the upload table to the appropriate field.
-  $limit = 500;
-  $result = db_query_range('SELECT DISTINCT u.fid, u.vid, u.list, u.description, n.nid, n.type FROM {upload} u INNER JOIN {node_revision} nr ON u.vid = nr.vid INNER JOIN {node} n ON n.nid = nr.nid WHERE u.vid > :lastvid ORDER BY u.vid, u.weight', 0, $limit, array(':lastvid' => $sandbox['last_vid_processed']));
+/**
+ * Migrate upload.module data to the newly created file field.
+ */
+function system_update_7061(&$sandbox) {
+  if (!db_table_exists('upload')) {
+    return;
+  }
+
+  if (!isset($sandbox['progress'])) {
+    // Retrieve a list of node revisions that have uploaded files attached.
+    // DISTINCT queries are expensive, especially when paged, so we store the
+    // data in its own table for the duration of the update.
+    $table = array(
+      'description' => t('Stores temporary data for system_update_7061.'),
+      'fields' => array('vid' => array('type' => 'int')),
+      'primary key' => array('vid'),
+    );
+    db_create_table('system_update_7061', $table);
+    $query = db_select('upload', 'u');
+    $query->distinct();
+    $query->addField('u','vid');
+    db_insert('system_update_7061')
+      ->from($query)
+      ->execute();
+
+    // Initialize batch update information.
+    $sandbox['progress'] = 0;
+    $sandbox['last_vid_processed'] = -1;
+    $sandbox['max'] = db_query("SELECT COUNT(*) FROM {system_update_7061}")->fetchField();
+  }
+
+  $node_revisions = array();
+
+  // Determine vids for this batch.
+  // Process all files attached to a given revision during the same batch.
+  $limit = 100;
+  $vids = db_query_range('SELECT vid FROM {system_update_7061} WHERE vid > :lastvid ORDER BY vid', 0, $limit, array(':lastvid' => $sandbox['last_vid_processed']))
+    ->fetchCol();
+
+  // Retrieve information on all the files attached to these revisions.
+  $result = db_query('SELECT u.fid, u.vid, u.list, u.description, n.nid, n.type, u.weight FROM {upload} u INNER JOIN {node_revision} nr ON u.vid = nr.vid INNER JOIN {node} n ON n.nid = nr.nid WHERE u.vid IN (:vids) ORDER BY u.vid, u.weight, u.fid', array(':vids' => $vids));
   foreach ($result as $record) {
-    // Note that we still reference the old files table here, since upload will
-    // not know about the new FID in the new file_managed table.
+    // For each uploaded file, retrieve the corresponding data from the old
+    // files table (since upload doesn't know about the new entry in the
+    // file_managed table).
     $file = db_select('files', 'f')
       ->fields('f', array('fid', 'uid', 'filename', 'filepath', 'filemime', 'filesize', 'status', 'timestamp'))
       ->condition('f.fid', $record->fid)
@@ -2721,35 +2758,30 @@ function system_update_7060(&$sandbox) {
       continue;
     }
 
+    // Add in the file information from the upload table.
     $file['description'] = $record->description;
     $file['display'] = $record->list;
 
+    // Create one record for each revision that contains all the uploaded files.
     $node_revisions[$record->vid]['nid'] = $record->nid;
     $node_revisions[$record->vid]['vid'] = $record->vid;
     $node_revisions[$record->vid]['type'] = $record->type;
     $node_revisions[$record->vid]['file'][LANGUAGE_NONE][] = $file;
   }
 
-  // To make sure we process an entire node all at once, toss the last node
-  // revision (which might be partial) unless it's the last one.
-  if ((count($node_revisions) > 1) && ($result->rowCount() == $limit)) {
-    array_pop($node_revisions);
-  }
-  else {
-    $finished = TRUE;
-  }
-
+  // Now that we know which files belong to which revisions, update the files'
+  // database entries, and save a reference to each file in the upload field on
+  // their node revisions.
   $basename = variable_get('file_directory_path', conf_path() . '/files');
   $scheme = file_default_scheme() . '://';
   foreach ($node_revisions as $vid => $revision) {
-    // We will convert filepaths to uri using the default scheme
-    // and stripping off the existing file directory path.
-    $fids = array();
     foreach ($revision['file'][LANGUAGE_NONE] as $delta => $file) {
-      // Insert into the file_managed table.
+      // We will convert filepaths to uri using the default scheme
+      // and stripping off the existing file directory path.
       $file['uri'] = $scheme . str_replace($basename, '', $file['filepath']);
       $file['uri'] = file_stream_wrapper_uri_normalize($file['uri']);
       unset($file['filepath']);
+      // Insert into the file_managed table.
       // Each fid should only be stored once in file_managed.
       db_merge('file_managed')
         ->key(array(
@@ -2766,26 +2798,34 @@ function system_update_7060(&$sandbox) {
         ))
         ->execute();
 
-      // Update the node field with the file URI.
-      $revision['file'][LANGUAGE_NONE][$delta] = $file;
-
       // Add the usage entry for the file.
       $file = (object) $file;
       file_usage_add($file, 'file', 'node', $revision['nid']);
+
+      // Update the node revision's upload file field with the file data.
+      $revision['file'][LANGUAGE_NONE][$delta] = array('fid' => $file->fid, 'display' => $file->display, 'description' => $file->description);
     }
 
-    // Insert the revision's files into the field_upload table.
+    // Write the revision's upload field data into the field_upload tables.
     $node = (object) $revision;
-    field_sql_storage_field_storage_write('node', $node, FIELD_STORAGE_INSERT, array($sandbox['field_id']));
+    _update_7000_field_sql_storage_write('node', $node->type, $node->nid, $node->vid, 'upload', $node->file);
 
     // Update our progress information for the batch update.
     $sandbox['progress']++;
     $sandbox['last_vid_processed'] = $vid;
   }
 
+  // If less than limit node revisions were processed, the update process is
+  // finished.
+  if (count($node_revisions) < $limit) {
+    $finished = TRUE;
+  }
+
+
   // If there's no max value then there's nothing to update and we're finished.
   if (empty($sandbox['max']) || isset($finished)) {
     db_drop_table('upload');
+    db_drop_table('system_update_7061');
     return t('Upload module has been migrated to File module.');
   }
   else {
@@ -2797,7 +2837,7 @@ function system_update_7060(&$sandbox) {
 /**
  * Replace 'system_list' index with 'bootstrap' index on {system}.
  */
-function system_update_7061() {
+function system_update_7062() {
   db_drop_index('system', 'bootstrap');
   db_drop_index('system', 'system_list');
   db_add_index('system', 'system_list', array('status', 'bootstrap', 'type', 'weight', 'name'));