- Rewrote the headline module from scratch. Note that the old

  headline code is still in place 'till the new code has proven
  to be stable. See "syndication.module" for the new code.

  Changes:

   + Improved the parser and tested it against RSS 0.9, RSS 0.91,
     RSS 0.92, RSS 1.0, RDF and XML feeds.

   + Improved the administration interface.  It might be a bit fuzzy
     at first.  Maybe some native English like Julian, Michael (or any
     one else with knowledge in the field) can help out by suggesting
     better naming, terminology or descriptions - as well as by
     writing the help section for this module?  I'd have no idea how
     much this would be appreciated.

   + We can *easily* recognize new tags or extensions: we parse out
     "link", "title", "description" and "author" right now, but we
     will have to revise which tags to support and which not.  New
     tags can be added in less than 10 minutes (if you are familiar
     with the code).  Read: we have something we can build on.

   + Within each item, tags can now appear is random order which is
     or was not the case with the old headline code where we expect
     <link>s prior to <description>s for example.

   + Feed updates only (ie. always) happen through cron.  Neither do
     we use one global cron for updating all feeds; instead, every
     feed can specify his own update-interval.

   + Newly fetched headlines are "appended" to the pool of existing
     headlines (read: we don't replace the whole feed), and headlines
     automatically "expire" after x days or hours.  (Every headline
     has a timestamp.)

   + Got rid of backend.class; it is integrated in the module.

   + Switched to more generic names: "headline" became "item" and
     "backend" became "feed".  This should ease future non-headline
     oriented syndication.

   + You can associate attributes or keyword lists with every feed.
     At the moment new items will automatically inherit their feeds
     attributes but in future we can use heuristics to make these
     attributes "mutate" when and where we see fit.  The attributes
     can be maintained by hand as well.

   + We don't export any blocks yet; we will soon do as soon this
     new code has been tested for a bit more.  We will only export
     bundles though so if you want to export by feed/source, you
     will have to make a source-specific bundle.

- Polished a bit on a few other modules: nothing major.

9 files changed, 313 insertions, 11 deletions

diff --git a/modules/forum.module b/modules/forum.module
index 031f8834b..7556d4315 100644
--- a/modules/forum.module
+++ b/modules/forum.module
@@ -74,7 +74,7 @@ function forum_admin() {
     case "edit":
       print forum_form(node_get_array(array(nid => $id)));
       break;
-   case t("Submit"):
+    case t("Submit"):
       print status(forum_save($edit));
       // fall through:
     default:
diff --git a/modules/forum/forum.module b/modules/forum/forum.module
index 031f8834b..7556d4315 100644
--- a/modules/forum/forum.module
+++ b/modules/forum/forum.module
@@ -74,7 +74,7 @@ function forum_admin() {
     case "edit":
       print forum_form(node_get_array(array(nid => $id)));
       break;
-   case t("Submit"):
+    case t("Submit"):
       print status(forum_save($edit));
       // fall through:
     default:
diff --git a/modules/index.module b/modules/index.module
index 0cc858c61..09c8e4abd 100644
--- a/modules/index.module
+++ b/modules/index.module
@@ -1,7 +1,10 @@
 <?php
 
+
+// entries => attributes
+
 function index_get_array($id) {
-  return db_fetch_array(db_query("SELECT * FROM entry WHERE eid = '$id'"));
+  return db_fetch_array(db_query("SELECT * FROM entry WHERE eid = '". check_input($id) ."'"));
 }
 
 function index_collection_form($name) {
@@ -29,9 +32,9 @@ function index_help() {
 function index_form($edit = array()) {
   global $REQUEST_URI;
 
-  $form .= form_textfield(t("Entry name"), "name", $edit[name], 35, 55, t("The name of this entry.  Example: 'Apache'."));
-  $form .= form_textfield(t("Collection"), "collection", $edit[collection], 35, 55, t("The collection or group this entry belgons to.  Example: 'Software'."));
-  $form .= form_textfield(t("Keywords"), "keyword", $edit[keyword], 35, 55, htmlentities("Format: <type>:<value>;<type>:<value>;.  Example: 'software:apache;type:webserver;os:linux;'."));
+  $form .= form_textfield(t("Entry name"), "name", $edit[name], 55, 64, t("The name of this entry.  Example: 'Apache'."));
+  $form .= form_textfield(t("Collection"), "collection", $edit[collection], 55, 64, t("The collection or group this entry belgons to.  Example: 'Software'."));
+  $form .= form_textfield(t("Keywords"), "keyword", $edit[keyword], 55, 64, htmlentities("Format: <type>:<value>;<type>:<value>;.  Example: 'software:apache;type:webserver;os:linux;'."));
   $form .= form_submit(t("Submit"));
 
   if ($edit[eid]) {
@@ -73,7 +76,7 @@ function index_test_1() {
   while ($entry = db_fetch_object($result)) {
     $form .= index_collection_form($entry->collection);
   }
-  $form .= "Select around and click the buttom below:<BR>";
+  $form .= "Select around and click the button below:<BR>";
   $form .= form_submit("Click to test");
 
   return form($REQUEST_URI, $form);
diff --git a/modules/page.module b/modules/page.module
index f9e59ed72..91ef7b8fc 100644
--- a/modules/page.module
+++ b/modules/page.module
@@ -71,7 +71,7 @@ function page_admin() {
     case "listing":
       print node_listing(page_query());
       break;
-   case t("Submit"):
+    case t("Submit"):
       print status(page_save($edit));
       // fall through:
     default:
diff --git a/modules/page/page.module b/modules/page/page.module
index f9e59ed72..91ef7b8fc 100644
--- a/modules/page/page.module
+++ b/modules/page/page.module
@@ -71,7 +71,7 @@ function page_admin() {
     case "listing":
       print node_listing(page_query());
       break;
-   case t("Submit"):
+    case t("Submit"):
       print status(page_save($edit));
       // fall through:
     default:
diff --git a/modules/story.module b/modules/story.module
index d4faf74d9..dfc5344c7 100644
--- a/modules/story.module
+++ b/modules/story.module
@@ -45,7 +45,6 @@ function story_form($edit = array()) {
   $form .= form_textarea(t("Abstract"), "abstract", $edit[abstract], 50, 10, t("Allowed HTML tags") .": ". htmlspecialchars(variable_get("allowed_html", "")));
   $form .= form_textarea(t("Body"), "body", $edit[body], 50, 15, t("Allowed HTML tags") .": ". htmlspecialchars(variable_get("allowed_html", "")));
 
-  // hidden fields:
   if ($edit[nid] > 0) {
     $form .= form_hidden("nid", $edit[nid]);
   }
diff --git a/modules/story/story.module b/modules/story/story.module
index d4faf74d9..dfc5344c7 100644
--- a/modules/story/story.module
+++ b/modules/story/story.module
@@ -45,7 +45,6 @@ function story_form($edit = array()) {
   $form .= form_textarea(t("Abstract"), "abstract", $edit[abstract], 50, 10, t("Allowed HTML tags") .": ". htmlspecialchars(variable_get("allowed_html", "")));
   $form .= form_textarea(t("Body"), "body", $edit[body], 50, 15, t("Allowed HTML tags") .": ". htmlspecialchars(variable_get("allowed_html", "")));
 
-  // hidden fields:
   if ($edit[nid] > 0) {
     $form .= form_hidden("nid", $edit[nid]);
   }
diff --git a/modules/syndication.module b/modules/syndication.module
new file mode 100644
index 000000000..3ea4bdc7d
--- /dev/null
+++ b/modules/syndication.module
@@ -0,0 +1,267 @@
+<?php
+
+include_once "modules/backend.class";
+
+function syndication_help() {
+ ?>
+  <P>TODO - anyone?</P>
+ <?php
+}
+
+function syndication_cron() {
+  $result = db_query("SELECT * FROM feed");
+  while ($feed = db_fetch_array($result)) {
+    // remove expired items:
+    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp - ". time() ." > $feed[uncache]");
+
+    // update feeds:
+    if ($feed[timestamp] + $feed[refresh] < time()) syndication_update($feed);
+  }
+}
+
+function syndication_bundle($feed) {
+  if ($feed->attribute) {
+    // compose query:
+    $keys = explode(",", $feed->attribute);
+    foreach ($keys as $key) $where[] = "attribute LIKE '%". trim($key) ."%'";
+
+    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where));
+
+    while ($item = db_fetch_object($result)) {
+      $output .= "<LI><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A></LI>";
+    }
+
+    return "<B>$feed->title</B><UL>$output</UL>";
+  }
+}
+
+function syndication_view_block() {
+  $result = db_query("SELECT * FROM bundle");
+  while ($bundle = db_fetch_object($result)) {
+    $output .= syndication_bundle($bundle);
+  }
+  return $output;
+}
+
+function syndication_update($feed) {
+
+  // open socket:
+  $url = parse_url($feed[link]);
+  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);
+
+  if ($fp) {
+    // fetch data:
+    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");
+    while(!feof($fp)) $data .= fgets($fp, 128);
+
+    if (strstr($data, "200 OK")) {
+
+      eregi("<item(.*)</item>", $data, $data);
+
+      foreach (explode("</item>", $data[0]) as $item) {
+        $l = eregi("<link>(.*)</link>", $item, $link);
+        $t = eregi("<title>(.*)</title>", $item, $title);
+        $a = eregi("<author>(.*)</author>", $item, $author);
+        $d = eregi("<description>(.*)</description>", $item, $description);
+
+        if ($l || $t || $a || $d) {
+          syndication_save_item(array(fid => $feed[fid], title => $title[0], link => $link[0], author => $author[0], description => $description[0], attribute => $feed[attribute]));
+        }
+      }
+
+      db_query("UPDATE feed SET timestamp = '". time() ."' WHERE fid = '". $feed[fid] ."'");
+    }
+    else {
+      watchdog("error", "failed to syndicate from '$feed[title]'");
+    }
+  }
+}
+
+function syndication_save_item($edit) {
+  if ($edit[iid] && $edit[title]) {
+    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attribute = '". check_input($edit[attribute]) ."' WHERE iid = '$edit[iid]'");
+  }
+  else if ($edit[iid]) {
+    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
+  }
+  else {
+    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
+      db_query("INSERT INTO item (fid, title, link, author, description, attribute, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attribute]) ."', '". time() ."')");
+    }
+  }
+}
+
+function syndication_form_bundle($edit = array()) {
+  global $REQUEST_URI;
+
+  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
+  $form .= form_textfield("Attributes", "attribute", $edit[attribute], 50, 128, "A comma-seperated list of keywords describing the bundle.");
+
+  $form .= form_submit("Submit");
+
+  if ($edit[bid]) {
+    $form .= form_submit(t("Delete"));
+    $form .= form_hidden("bid", $edit[bid]);
+  }
+
+  return form($REQUEST_URI, $form);
+}
+
+function syndication_save_bundle($edit) {
+  if ($edit[bid] && $edit[title]) {
+    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attribute = '". check_input($edit[attribute]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
+  }
+  else if ($edit[bid]) {
+    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
+  }
+  else {
+    db_query("INSERT INTO bundle (title, attribute) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attribute]) ."')");
+  }
+}
+
+function syndication_form_feed($edit = array()) {
+  global $REQUEST_URI;
+
+  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));
+
+  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
+  $form .= form_textfield("Link", "link", $edit[link], 50, 64, "The fully-qualified URL of the feed.");
+  $form .= form_textfield("Attributes", "attribute", $edit[attribute], 50, 128, "A comma-seperated list of keywords describing the feed.");
+  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
+  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");
+
+  $form .= form_submit("Submit");
+
+  if ($edit[fid]) {
+    $form .= form_submit(t("Delete"));
+    $form .= form_hidden("fid", $edit[fid]);
+  }
+
+  return form($REQUEST_URI, $form);
+}
+
+function syndication_save_feed($edit) {
+  if ($edit[fid] && $edit[title]) {
+    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', attribute = '". check_input($edit[attribute]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
+    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
+  }
+  else if ($edit[fid]) {
+    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
+    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
+  }
+  else {
+    db_query("INSERT INTO feed (title, link, attribute, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[attribute]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
+  }
+}
+
+function syndication_save_attributes($edit) {
+  foreach($edit as $iid => $value) {
+    db_query("UPDATE item SET attribute = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
+  }
+  return "attributes has been saved";
+}
+
+function syndication_get_feed($fid) {
+  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
+}
+
+function syndication_get_bundle($bid) {
+  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
+}
+
+function syndication_view_feed() {
+  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");
+
+  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
+  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>items</TH><TH>last update</TH><TH>next update</TH><TH COLSPAN=\"2\">operations</TH></TR>\n";
+  while ($feed = db_fetch_object($result)) {
+    $output .= " <TR><TD>". check_output($feed->title) ."</TD><TD>". check_output($feed->attribute) ."</TD><TD>". format_plural($feed->items, "item", "items") ."</TD><TD>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</TD><TD>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</TD><TD><A HREF=\"admin.php?mod=syndication&type=feed&op=edit&id=$feed->fid\">edit feed</A></TD><TD><A HREF=\"admin.php?mod=syndication&type=feed&op=update&id=$feed->fid\">update items</A></TD></TR>\n";
+  }
+  $output .= "</TABLE>\n";
+
+  return $output;
+}
+
+function syndication_view_bundle() {
+  $result = db_query("SELECT * FROM bundle ORDER BY title");
+
+  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
+  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>operations</TH></TR>\n";
+  while ($bundle = db_fetch_object($result)) {
+    $output .= " <TR><TD>". check_output($bundle->title) ."</TD><TD>". check_output($bundle->attribute) ."</TD><TD><A HREF=\"admin.php?mod=syndication&type=bundle&op=edit&id=$bundle->bid\">edit bundle</A></TD></TR>\n";
+  }
+  $output .= "</TABLE>\n";
+
+  return $output;
+}
+
+function syndication_view_item() {
+  global $REQUEST_URI;
+
+  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");
+
+  $output .= "<FORM ACTION=\"$REQUEST_URI\" METHOD=\"post\">\n";
+  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
+  $output .= " <TR><TH>time</TH><TH>feed</TH><TH>item</TH></TR>\n";
+  while ($item = db_fetch_object($result)) {
+    $output .= " <TR><TD VALIGN=\"top\" NOWRAP>". format_date($item->timestamp, "custom", "H:i") ."</TD><TD VALIGN=\"top\" NOWRAP><A HREF=\"admin.php?mod=syndication&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</A></TD><TD><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A>". ($item->description ? "<BR><SMALL><I>". check_output($item->description) ."</I></SMALL>" : "") ."<BR><INPUT TYPE=\"text\" NAME=\"edit[$item->iid]\" VALUE=\"". check_form($item->attribute) ."\" SIZE=\"50\"></TD></TR>\n";
+  }
+  $output .= "</TABLE>\n";
+  $output .= "<INPUT TYPE=\"submit\" NAME=\"op\" VALUE=\"Save attributes\">\n";
+  $output .= "</FORM>\n";
+
+  return $output;
+}
+
+function syndication_admin() {
+  global $op, $id, $type, $edit;
+
+  print "<SMALL><A HREF=\"admin.php?mod=syndication&type=bundle&op=add\">add new bundle</A> | <A HREF=\"admin.php?mod=syndication&type=feed&op=add\">add new feed</A> | <A HREF=\"admin.php?mod=syndication&op=block\">available blocks</A> | <A HREF=\"admin.php?mod=syndication&type=bundle&op=view\">bundle overview</A> | <A HREF=\"admin.php?mod=syndication&type=feed&op=view\">feed overview</A> | <A HREF=\"admin.php?mod=syndication&type=item&op=view\">item overview</A> | <A HREF=\"admin.php?mod=syndication&op=help\">help</A></SMALL><HR>";
+
+  switch($op) {
+    case "help":
+      print syndication_help();
+      break;
+    case "add":
+      if ($type == "bundle")
+        print syndication_form_bundle();
+      else
+        print syndication_form_feed();
+      break;
+    case "block":
+      print syndication_view_block();
+      break;
+    case "edit":
+      if ($type == "bundle")
+        print syndication_form_bundle(syndication_get_bundle($id));
+      else
+        print syndication_form_feed(syndication_get_feed($id));
+      break;
+    case "update":
+      print syndication_update(syndication_get_feed($id));
+      print syndication_view_feed();
+      break;
+    case "Save attributes":
+      print status(syndication_save_attributes($edit));
+      print syndication_view_item();
+      break;
+    case "Delete":
+      $edit[title] = 0;
+      // fall through:
+    case "Submit":
+      if ($type == "bundle")
+        print status(syndication_save_bundle($edit));
+      else
+        print status(syndication_save_feed($edit));
+      // fall through:
+    default:
+      if ($type == "bundle")
+        print syndication_view_bundle();
+      else if ($type == "item")
+        print syndication_view_item();
+      else
+        print syndication_view_feed();
+  }
+}
+
+?>
diff --git a/updates/2.00-to-x.xx.sql b/updates/2.00-to-x.xx.sql
index 3e17e9f46..a60ab2cb2 100644
--- a/updates/2.00-to-x.xx.sql
+++ b/updates/2.00-to-x.xx.sql
@@ -152,5 +152,39 @@ CREATE TABLE entry (
   name varchar(32) DEFAULT '' NOT NULL,
   keyword varchar(255) DEFAULT '' NOT NULL,
   collection varchar(32) DEFAULT '' NOT NULL,
+  UNIQUE name (name, collection),
   PRIMARY KEY (eid)
 );
+
+CREATE TABLE bundle (
+  bid int(11) DEFAULT '0' NOT NULL auto_increment,
+  title varchar(255) DEFAULT '' NOT NULL,
+  attribute varchar(255) DEFAULT '' NOT NULL,
+  UNIQUE (title),
+  PRIMARY KEY (bid)
+);
+
+CREATE TABLE feed (
+  fid int(11) DEFAULT '0' NOT NULL auto_increment,
+  title varchar(255) DEFAULT '' NOT NULL,
+  link varchar(255) DEFAULT '' NOT NULL,
+  refresh int(11),
+  uncache int(11),
+  timestamp int(11),
+  attribute varchar(255) DEFAULT '' NOT NULL,
+  UNIQUE (title),
+  UNIQUE (link),
+  PRIMARY KEY (fid)
+);
+
+CREATE TABLE item (
+  iid int(11) DEFAULT '0' NOT NULL auto_increment,
+  fid int(11) DEFAULT '0' NOT NULL,
+  title varchar(255) DEFAULT '' NOT NULL,
+  link varchar(255) DEFAULT '' NOT NULL,
+  author varchar(255) DEFAULT '' NOT NULL,
+  description TEXT DEFAULT '' NOT NULL,
+  timestamp int(11),
+  attribute varchar(255) DEFAULT '' NOT NULL,
+  PRIMARY KEY (iid)
+);