#517814 by jmstacey, justinrandell, pwolanin, drewish, Jody Lynn, aaron, dopry, and c960657: Converted File API to stream wrappers, for enhanced private/public file handling, and the ability to reference other storage mechanisms such as s3:// and flicker://.

3 files changed, 75 insertions, 19 deletions

diff --git a/modules/upload/upload.install b/modules/upload/upload.install
index a4d22bea6..395abfebf 100644
--- a/modules/upload/upload.install
+++ b/modules/upload/upload.install
@@ -43,7 +43,7 @@ function upload_schema() {
         'unsigned' => TRUE,
         'not null' => TRUE,
         'default' => 0,
-        'description' => 'Primary Key: The {files}.fid.',
+        'description' => 'Primary Key: The {file}.fid.',
       ),
       'nid' => array(
         'type' => 'int',
@@ -98,3 +98,58 @@ function upload_schema() {
 }
 
 
+/**
+ * Migrate upload module files from {files} to {file}.
+ */
+function upload_update_7000(&$sandbox) {
+  $ret = array();
+
+  /*
+    TODO: Fix the updates. This is broken. See http://drupal.org/node/329301#comment-1404336
+    Also note new DB structure http://drupal.org/node/227232#comment-1683976
+  */
+
+  if (!isset($sandbox['progress'])) {
+    // Initialize batch update information.
+    $sandbox['progress'] = 0;
+    $sandbox['last_fid_processed'] = -1;
+    $sandbox['max'] = db_query("SELECT COUNT(DISTINCT u.fid) FROM {upload} u")->fetchField();
+  }
+
+  // As a batch operation move records from {files} into the {file} table.
+  $limit = 500;
+  $result = db_query_range("SELECT DISTINCT u.fid FROM {upload} u ORDER BY u.vid", array(), 0, $limit);
+  foreach ($result as $record) {
+    $old_file = db_query('SELECT f.* FROM {files} f WHERE f.fid = :fid', array(':fid' => $record->fid))->fetch(PDO::FETCH_OBJ);
+    if (!$old_file) {
+      continue;
+    }
+
+    $new_file = db_query('SELECT f.* FROM {files} f WHERE f.filepath = :filepath', array(':filepath' => $old_file->uri))->fetch(PDO::FETCH_OBJ);
+    if (!$new_file) {
+      // Re-save the file into the new {file} table.
+      $new_file = clone $old_file;
+      drupal_write_record('file', $new_file);
+    }
+
+    // If the fid has changed we need to update the {upload} record to use the
+    // new id.
+    if (!empty($new_file->fid) && ($new_file->fid != $old_file->fid)) {
+      db_update('upload')
+        ->fields(array('fid' => $new_file->fid))
+        ->condition('fid', $old_file->fid)
+        ->execute();
+    }
+
+    // Update our progress information for the batch update.
+    $sandbox['progress']++;
+    $sandbox['last_fid_processed'] = $old_file->fid;
+  }
+
+  // Indicate our current progress to the batch update system. If there's no
+  // max value then there's nothing to update and we're finished.
+  $ret['#finished'] = empty($sandbox['max']) ? 1 : ($sandbox['progress'] / $sandbox['max']);
+
+  return $ret;
+}
+
diff --git a/modules/upload/upload.module b/modules/upload/upload.module
index 772e329ed..eec81dbe7 100644
--- a/modules/upload/upload.module
+++ b/modules/upload/upload.module
@@ -150,8 +150,7 @@ function _upload_file_limits($user) {
  * Implement hook_file_download().
  */
 function upload_file_download($filepath) {
-  $filepath = file_create_path($filepath);
-  $file = db_query("SELECT f.*, u.nid FROM {files} f INNER JOIN {upload} u ON f.fid = u.fid WHERE filepath = :path", array(':path' => $filepath))->fetchObject();
+  $file = db_query("SELECT f.*, u.nid FROM {file} f INNER JOIN {upload} u ON f.fid = u.fid WHERE uri = :path", array(':path' => $filepath))->fetchObject();
 
   if ($file && user_access('view uploaded files') && ($node = node_load($file->nid)) && node_access('view', $node)) {
     return array(
@@ -182,7 +181,7 @@ function upload_node_form_submit(&$form, &$form_state) {
   );
 
   // Save new file uploads.
-  if (user_access('upload files') && ($file = file_save_upload('upload', $validators, file_directory_path()))) {
+  if (user_access('upload files') && ($file = file_save_upload('upload', $validators, 'public://'))) {
     $file->list = variable_get('upload_list_default', 1);
     $file->description = $file->filename;
     $file->weight = 0;
@@ -245,9 +244,9 @@ function upload_form_alter(&$form, $form_state, $form_id) {
       // Make sure necessary directories for upload.module exist and are
       // writable before displaying the attachment form.
       $path = file_directory_path();
-      $temp = file_directory_temp();
+      $temp = file_directory_path('temporary');
       // Note: pass by reference
-      if (!file_check_directory($path, FILE_CREATE_DIRECTORY) || !file_check_directory($temp, FILE_CREATE_DIRECTORY)) {
+      if (!file_prepare_directory($path, FILE_CREATE_DIRECTORY) || !file_prepare_directory($temp, FILE_CREATE_DIRECTORY)) {
         $form['attachments']['#description'] =  t('File attachments are disabled. The file directories have not been properly configured.');
         if (user_access('administer site configuration')) {
           $form['attachments']['#description'] .= ' ' . t('Please visit the <a href="@admin-file-system">file system configuration page</a>.', array('@admin-file-system' => url('admin/settings/file-system')));
@@ -372,7 +371,7 @@ function upload_node_view($node, $build_mode) {
       $node->rss_elements[] = array(
         'key' => 'enclosure',
         'attributes' => array(
-          'url' => file_create_url($file->filepath),
+          'url' => file_create_url($file->uri),
           'length' => $file->filesize,
           'type' => $file->filemime
         )
@@ -443,7 +442,7 @@ function theme_upload_attachments($elements) {
   foreach ($elements['#files'] as $file) {
     $file = (object)$file;
     if ($file->list && empty($file->remove)) {
-      $href = file_create_url($file->filepath);
+      $href = file_create_url($file->uri);
       $text = $file->description ? $file->description : $file->filename;
       $rows[] = array(l($text, $href), format_size($file->filesize));
     }
@@ -472,7 +471,7 @@ function upload_space_used($uid) {
  *   The amount of disk space used by uploaded files in bytes.
  */
 function upload_total_space_used() {
-  return db_query('SELECT SUM(f.filesize) FROM {files} f INNER JOIN {upload} u ON f.fid = u.fid')->fetchField();
+  return db_query('SELECT SUM(f.filesize) FROM {file} f INNER JOIN {upload} u ON f.fid = u.fid')->fetchField();
 }
 
 function upload_save($node) {
@@ -543,13 +542,13 @@ function _upload_form($node) {
       $file = (object)$file;
       $key = $file->fid;
 
-      $form['files'][$key]['description'] = array('#type' => 'textfield', '#default_value' => !empty($file->description) ? $file->description : $file->filename, '#maxlength' => 256, '#description' => '<small>' . file_create_url($file->filepath) . '</small>');
+      $form['files'][$key]['description'] = array('#type' => 'textfield', '#default_value' => !empty($file->description) ? $file->description : $file->filename, '#maxlength' => 256, '#description' => '<small>' . file_create_url($file->uri) . '</small>');
       $form['files'][$key]['size'] = array('#markup' => format_size($file->filesize));
       $form['files'][$key]['remove'] = array('#type' => 'checkbox', '#default_value' => !empty($file->remove));
       $form['files'][$key]['list'] = array('#type' => 'checkbox',  '#default_value' => $file->list);
       $form['files'][$key]['weight'] = array('#type' => 'weight', '#delta' => count($node->files), '#default_value' => $file->weight);
       $form['files'][$key]['filename'] = array('#type' => 'value',  '#value' => $file->filename);
-      $form['files'][$key]['filepath'] = array('#type' => 'value',  '#value' => $file->filepath);
+      $form['files'][$key]['uri'] = array('#type' => 'value',  '#value' => $file->uri);
       $form['files'][$key]['filemime'] = array('#type' => 'value',  '#value' => $file->filemime);
       $form['files'][$key]['filesize'] = array('#type' => 'value',  '#value' => $file->filesize);
       $form['files'][$key]['fid'] = array('#type' => 'value',  '#value' => $file->fid);
diff --git a/modules/upload/upload.test b/modules/upload/upload.test
index bf9b22bbb..da0aafe6a 100644
--- a/modules/upload/upload.test
+++ b/modules/upload/upload.test
@@ -45,7 +45,7 @@ class UploadTestCase extends DrupalWebTestCase {
     // Create a node and attempt to attach files.
     $node = $this->drupalCreateNode();
     $text_files = $this->drupalGetTestFiles('text');
-    $files = array(current($text_files)->filepath, next($text_files)->filepath);
+    $files = array(current($text_files)->uri, next($text_files)->uri);
 
     $this->uploadFile($node, $files[0]);
     $this->uploadFile($node, $files[1]);
@@ -58,7 +58,7 @@ class UploadTestCase extends DrupalWebTestCase {
     $this->checkUploadedFile(basename($files[1]));
 
     // Check that files are also accessible when using private files.
-    variable_set('file_downloads', FILE_DOWNLOADS_PRIVATE);
+    variable_set('file_default_scheme', 'private');
     $this->checkUploadedFile(basename($files[0]));
     $this->checkUploadedFile(basename($files[1]));
 
@@ -86,7 +86,9 @@ class UploadTestCase extends DrupalWebTestCase {
       $this->assertRaw(t('Page %title has been updated.', array('%title' => $node->title)), 'File deleted successfully.');
 
       $this->assertNoText($new_name, $new_name . ' not found on node.');
-      $this->drupalGet($base_url . '/' . file_directory_path() . '/' . $upload->description, array('external' => TRUE));
+      $uri = 'public://' . $upload->description;
+      $external_uri = file_stream_wrapper_get_instance_by_uri($uri)->getExternalUrl();
+      $this->drupalGet($external_uri, array('external' => TRUE));
       $this->assertResponse(array(404), 'Uploaded ' . $upload->description . ' is not accessible.');
     }
     else {
@@ -119,17 +121,17 @@ class UploadTestCase extends DrupalWebTestCase {
     $text_file = current($this->drupalGetTestFiles('text'));
     // Select a file that's less than the 1MB upload limit so we only test one
     // limit at a time.
-    $this->uploadFile($node, $text_file->filepath, FALSE);
+    $this->uploadFile($node, $text_file->uri, FALSE);
     // Test the error message in two steps in case there are additional errors
     // that change the error message's format.
-    $this->assertRaw(t('The specified file %name could not be uploaded.', array('%name' => $text_file->filename)), t('File %filepath was not allowed to be uploaded', array('%filepath' => $text_file->filepath)));
+    $this->assertRaw(t('The specified file %name could not be uploaded.', array('%name' => $text_file->filename)), t('File %filepath was not allowed to be uploaded', array('%filepath' => $text_file->uri)));
     $this->assertRaw(t('Only files with the following extensions are allowed: %files-allowed.', array('%files-allowed' => $settings['upload_extensions'])), t('File extension cited as reason for failure'));
 
     // Attempt to upload .html file when .html is only extension allowed.
     $html_files = array_values($this->drupalGetTestFiles('html'));
     // Use the HTML file with the .html extension, $html_files[0] has a .txt
     // extension.
-    $html_file = $html_files[1]->filepath;
+    $html_file = $html_files[1]->uri;
     $this->uploadFile($node, $html_file);
     $this->assertNoRaw(t('The specified file %name could not be uploaded.', array('%name' => basename($html_file))), t('File ' . $html_file . ' was allowed to be uploaded'));
   }
@@ -139,7 +141,7 @@ class UploadTestCase extends DrupalWebTestCase {
    */
   function testLimit() {
     $files = $this->drupalGetTestFiles('text', 1310720); // 1 MB.
-    $file = current($files)->filepath;
+    $file = current($files)->uri;
 
     $admin_user = $this->drupalCreateUser(array('administer site configuration'));
     $web_user = $this->drupalCreateUser(array('access content', 'edit own page content', 'upload files', 'view uploaded files'));
@@ -206,7 +208,7 @@ class UploadTestCase extends DrupalWebTestCase {
    */
   function checkUploadedFile($filename) {
     global $base_url;
-    $file = file_directory_path() . '/' . $filename;
+    $file = 'public://' . $filename;
     $this->drupalGet(file_create_url($file), array('external' => TRUE));
     $this->assertResponse(array(200), 'Uploaded ' . $filename . ' is accessible.');
     $this->assertTrue(strpos($this->drupalGetHeader('Content-Type'), 'text/plain') === 0, t('MIME type is text/plain.'));