Merge branch 'master' into tdfischer/event-cachingtdfischer/event-caching

Conflicts:
	.gitignore
	src/db-reader.vala

35 files changed, 1944 insertions, 519 deletions

diff --git a/.bzrignore b/.gitignore
index ec29ed59..ba42b0fd 100644
--- a/.bzrignore
+++ b/.gitignore
@@ -69,3 +69,8 @@ mimetype-test
 marshalling-test
 test/direct/datamodel-test
 test/direct/event-cache-test
+libzeitgeist/*.c
+*.o
+*.la
+*.stamp
+*.lo
diff --git a/NEWS b/NEWS
index bb9daf17..e41bf855 100644
--- a/NEWS
+++ b/NEWS
@@ -1,10 +1,42 @@
-2012-03-01: Zeitgeist Bluebird Beta 1
+2012-04-10: Zeitgeist 0.9
+-------------------------------------
+
+Engine:
+ - Updated the list of recognized MIME-types.
+ - Set a size limit (4MiB) on the results returned by individual FindEvents and
+   GetEvents calls. Too big queries will result in an exception.
+ - Improved handling of FTS not being present (Beta 1 would leak memory).
+ - Enhanced database corruption detection and recovery.
+ - DataSourceRegistry: don't ignore bus addresses that registered more than
+   one data-source if one of them is disabled.
+
+FTS:
+ - Added a build-time option to disable FTS.
+ - The index is now automatically re-build if it gets corrupted.
+ - The index is now automatically re-build if the Zeitgeist DB is re-created.
+ - Split the reindex query into several smaller ones, for enhanced performance.
+ - Quit when Zeitgeist leaves the bus.
+ - Ignore events from Ubuntu One.
+
+Python API:
+ - Updated the list of recognized MIME-types.
+
+2012-03-20: Zeitgeist Bluebird Beta 1
 -------------------------------------
 
 Engine:
  - Fixed crash when events had NULL fields (LP: #941530).
- - Made sure no aborted SQLite transactions are left open.
+ - Made sure no aborted SQLite transactions are left open (LP: #937991).
  - Enhanced logging and added --log-file option.
+ - Fixed bug Storage Monitor corrupting storage IDs in DB (LP: #950983).
+ - Fixed bug inserting previously non-existant values when searching for
+   them (LP: #953041).
+
+FTS:
+ - Filter indexing of unrecognized values (eg. non-standard URIs).
+ - Enhanced grouping by URI and origin (LP: #947835).
+ - Use current_uri instead of URI; index MOVE_EVENTs (LP: #948794).
+ - Added SearchWithRelevancies D-Bus function.
 
 2012-02-14: Zeitgeist Bluebird Alpha 3
 --------------------------------------
@@ -195,37 +227,37 @@ Overall:
 
 Engine:
 
- - Added some new mimetype mappings
- - Speed up the GetEvents method by caching events (LP: #686732)
- - Allow extension without public method (LP: #691660)
+ - Added some new mimetype mappings.
+ - Speed up the GetEvents method by caching events (LP: #686732).
+ - Allow extension without public method (LP: #691660).
  - Added a read-only DBus property "extensions" to org.gnome.zeitgeist.Log 
-   (LP: #693861)
- - Added helper function to get an extension's name
+   (LP: #693861).
+ - Added helper function to get an extension's name.
  - Fixed bug in RemoteInterface.Quit(): make sure to close connection to the
    used bus if this method gets called, this is needed if RemoteInterface
    does not know anything about the loop it is running in.
- - Fix sending payload from engine to client (LP: #692645)
- - Improve performance of DataSourceRegistry (LP: #695311)
+ - Fix sending payload from engine to client (LP: #692645).
+ - Improve performance of DataSourceRegistry (LP: #695311).
  - Improve performance of find_event queries with timerange other than 
-   TimeRange.always() (LP: #672965)
- - Add an auto-filter for broken database entries (LP: #598666 workaround)
+   TimeRange.always() (LP: #672965).
+ - Add an auto-filter for broken database entries (LP: #598666 workaround).
  - Introduce new DB schema (version 3) that bundles a few performance related 
-   fixes (LP: #673452, #673394)
+   fixes (LP: #673452, #673394).
  - Added a (LRU-type) event cache to speed up similar requests.
 
 Python API:
 
- - Fix find_event_for_template to stop ignoring the template (LP: #690377)
- - Add get_extensions method to ZeitgeistClient
+ - Fix find_event_for_template to stop ignoring the template (LP: #690377).
+ - Add get_extensions method to ZeitgeistClient.
 
 Overall:
 
- - Using logging output for debugging purposes (LP: #660440)
- - Stop building man page for zeitgeist-datahub (LP: #680360)
- - Allow easy building for KDE (LP: #683280)
+ - Using logging output for debugging purposes (LP: #660440).
+ - Stop building man page for zeitgeist-datahub (LP: #680360).
+ - Allow easy building for KDE (LP: #683280).
  - Logging output now displays how many events were actually inserted from 
-   the batch of requested inserts (LP: #660440)
- - Changed License to LGPL 2.1+
+   the batch of requested inserts (LP: #660440).
+ - Changed License to LGPL 2.1+.
 
 2010-11-01: Zeitgeist 0.6 "Buzzer Beater"
 -----------------------------------------
diff --git a/configure.ac b/configure.ac
index 4a21951b..fc0b2029 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT([zeitgeist], [0.8.99~alpha3], [dev@lists.zeitgeist-project.com], [zeitgeist])
+AC_INIT([zeitgeist], [0.9.0], [dev@lists.zeitgeist-project.com], [zeitgeist])
 AC_CONFIG_SRCDIR([Makefile.am])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([dist-bzip2 no-dist-gzip])
@@ -80,6 +80,16 @@ fi
 DBUS_SERVICES_DIR="$services_dir"
 AC_SUBST(DBUS_SERVICES_DIR)
 
+
+#################################################
+# Configure options: --disable-fts.
+#################################################
+AC_ARG_ENABLE([fts],
+            AS_HELP_STRING([--disable-fts], [Build without FTS++ extension]),
+            [enable_fts=$enableval],[enable_fts="yes"])
+
+AM_CONDITIONAL(HAVE_FTS, test "x$enable_fts" != "xno")
+
 AC_CONFIG_FILES([
 	Makefile
 	src/Makefile
@@ -131,6 +141,7 @@ ${PACKAGE}-${VERSION}
     Install Prefix:     ${prefix}
 
   Optional dependencies
+    fts++:              ${enable_fts}
     dee-icu:            ${with_dee_icu}
 
 EOF
diff --git a/extensions/Makefile.am b/extensions/Makefile.am
index d5ed32db..060f36b8 100644
--- a/extensions/Makefile.am
+++ b/extensions/Makefile.am
@@ -1,4 +1,6 @@
+if HAVE_FTS
 SUBDIRS = fts++
+endif
 
 NULL =
 
diff --git a/extensions/blacklist.vala b/extensions/blacklist.vala
index a872b9ca..ec69969d 100644
--- a/extensions/blacklist.vala
+++ b/extensions/blacklist.vala
@@ -37,9 +37,9 @@ namespace Zeitgeist
             throws Error;
 
         public signal void template_added (string template_id,
-            [DBus (signature = "s(asaasay)")] Variant event_template);
+            [DBus (signature = "(asaasay)")] Variant event_template);
         public signal void template_removed (string template_id,
-            [DBus (signature = "s(asassay)")] Variant event_template);
+            [DBus (signature = "(asassay)")] Variant event_template);
     }
 
     namespace BlacklistTemplates
@@ -47,7 +47,7 @@ namespace Zeitgeist
         private const string SIG_BLACKLIST = "a{s("+Utils.SIG_EVENT+")}";
 
         private static HashTable<string, Event> from_variant (
-            Variant templates_variant)
+            Variant templates_variant) throws EngineError
         {
             var blacklist = new HashTable<string, Event> (str_hash, str_equal);
 
@@ -100,9 +100,21 @@ namespace Zeitgeist
             Variant? templates = retrieve_config ("blacklist",
                 BlacklistTemplates.SIG_BLACKLIST);
             if (templates != null)
-                blacklist = BlacklistTemplates.from_variant (templates);
+            {
+                try
+                {
+                    blacklist = BlacklistTemplates.from_variant (templates);
+                }
+                catch (EngineError e)
+                {
+                    warning ("Could not load blacklist from variant: %s", e.message);
+                    blacklist = new HashTable<string, Event> (str_hash, str_equal);
+                }
+            }
             else
+            {
                 blacklist = new HashTable<string, Event> (str_hash, str_equal);
+            }
 
             // This will be called after bus is acquired, so it shouldn't block
             try
diff --git a/extensions/ds-registry.vala b/extensions/ds-registry.vala
index c651d039..46beb374 100644
--- a/extensions/ds-registry.vala
+++ b/extensions/ds-registry.vala
@@ -72,7 +72,7 @@ namespace Zeitgeist
         }
 
         public DataSource.from_variant (Variant variant,
-            bool reset_running=false)
+            bool reset_running=false) throws EngineError
         {
             warn_if_fail (
                 variant.get_type_string () == "(sssa("+Utils.SIG_EVENT+")bxb)"
@@ -127,7 +127,7 @@ namespace Zeitgeist
             "a(sssa("+Utils.SIG_EVENT+")bxb)";
 
         private static HashTable<string, DataSource> from_variant (
-            Variant sources_variant, bool reset_running=false)
+            Variant sources_variant, bool reset_running=false) throws EngineError
         {
             var registry = new HashTable<string, DataSource> (
                 str_hash, str_equal);
@@ -166,8 +166,10 @@ namespace Zeitgeist
 
     class DataSourceRegistry: Extension, RemoteRegistry
     {
+        private const string MULTIPLE_MARKER = "<multiple>";
         private HashTable<string, DataSource> sources;
-        private HashTable<string, GenericArray<BusName>> running;
+        private HashTable<string, GenericArray<BusName>> running_ds;
+        private HashTable<string, string> bus_name_2_ds;
         private uint registration_id;
         private bool dirty;
 
@@ -180,16 +182,30 @@ namespace Zeitgeist
 
         construct
         {
-            running = new HashTable<string, GenericArray<BusName?>>(
+            bus_name_2_ds = new HashTable<string, string> (str_hash, str_equal);
+            running_ds = new HashTable<string, GenericArray<BusName?>>(
                 str_hash, str_equal);
 
             Variant? registry = retrieve_config ("registry",
                 DataSources.SIG_DATASOURCES);
             if (registry != null)
-                sources = DataSources.from_variant (registry, true);
+            {
+                try
+                {
+                    sources = DataSources.from_variant (registry, true);
+                }
+                catch (EngineError e)
+                {
+                    warning ("Error while loading datasource registry: %s", e.message);
+                    sources = new HashTable<string, DataSource> (
+                        str_hash, str_equal);
+                }
+            }
             else
+            {
                 sources = new HashTable<string, DataSource> (
                     str_hash, str_equal);
+            }
 
             // this will be called after bus is acquired, so it shouldn't block
             try
@@ -249,7 +265,7 @@ namespace Zeitgeist
         }
 
         public bool register_data_source (string unique_id, string name,
-            string description, Variant event_templates, BusName? sender)
+            string description, Variant event_templates, BusName? sender) throws EngineError
         {
             debug ("%s: %s, %s, %s", Log.METHOD, unique_id, name, description);
             if (sender == null)
@@ -259,15 +275,26 @@ namespace Zeitgeist
             }
 
 
-            var sender_array = running.lookup (unique_id);
+            var sender_array = running_ds.lookup (unique_id);
             if (sender_array == null)
             {
-                running.insert (unique_id, new GenericArray<BusName?>());
-                running.lookup (unique_id).add (sender);
+                sender_array = new GenericArray<BusName?>();
+                sender_array.add (sender);
+                running_ds.insert (unique_id, sender_array);
+            }
+            else if (!is_sender_known (sender, sender_array))
+            {
+                sender_array.add (sender);
+            }
+
+            unowned string ds_id = bus_name_2_ds.lookup (sender);
+            if (ds_id == null)
+            {
+                bus_name_2_ds.insert (sender, unique_id);
             }
-            else if (is_sender_known (sender, sender_array))
+            else if (ds_id != unique_id && ds_id != MULTIPLE_MARKER)
             {
-                running.lookup (unique_id).add (sender);
+                bus_name_2_ds.insert (sender, MULTIPLE_MARKER);
             }
 
             unowned DataSource? ds = sources.lookup (unique_id);
@@ -337,17 +364,21 @@ namespace Zeitgeist
         public override void pre_insert_events (GenericArray<Event?> events,
             BusName? sender)
         {
-            foreach (string unique_id in running.get_keys())
+            foreach (unowned string unique_id in running_ds.get_keys())
             {
-                GenericArray<BusName?> bus_names = running.lookup (unique_id);
+                GenericArray<BusName?> bus_names = running_ds.lookup (unique_id);
                 if (is_sender_known (sender, bus_names))
                 {
                     var data_source = sources.lookup (unique_id);
 
-                    data_source.timestamp =  Timestamp.now ();
+                    data_source.timestamp = Timestamp.now ();
                     dirty = true;
 
-                    if (!data_source.enabled)
+                    // if one sender registers multiple unique data sources,
+                    // we have to rely that it's the correct thing, otherwise
+                    // we can just ignore the events
+                    unowned string ds_id = bus_name_2_ds.lookup (sender);
+                    if (!data_source.enabled && ds_id != MULTIPLE_MARKER)
                     {
                         for (int i = 0; i < events.length; i++)
                             events[i] = null;
@@ -373,7 +404,7 @@ namespace Zeitgeist
             var disconnected_ds = new GenericArray<DataSource> ();
             {
                 var iter = HashTableIter<string, GenericArray<BusName?>> (
-                    running);
+                    running_ds);
                 unowned string uid;
                 unowned GenericArray<BusName> name_arr;
                 while (iter.next (out uid, out name_arr))
@@ -401,11 +432,11 @@ namespace Zeitgeist
                 ds.timestamp = Timestamp.now ();
                 dirty = true;
 
-                if (running.lookup (uid).length == 0)
+                if (running_ds.lookup (uid).length == 0)
                 {
                     debug ("No remaining client running: %s [%s]",
                         ds.name, uid);
-                    running.remove (uid);
+                    running_ds.remove (uid);
                     ds.running = false;
 
                     data_source_disconnected (ds.to_variant ());
diff --git a/extensions/fts++/controller.cpp b/extensions/fts++/controller.cpp
index d945a327..e8ce8f52 100644
--- a/extensions/fts++/controller.cpp
+++ b/extensions/fts++/controller.cpp
@@ -37,19 +37,35 @@ void Controller::Run ()
 void Controller::RebuildIndex ()
 {
   GError *error = NULL;
-  GPtrArray *events;
-  GPtrArray *templates = g_ptr_array_new ();
+  guint32 *event_ids;
+  gint event_ids_size;
+  GPtrArray *templates = g_ptr_array_new_with_free_func (g_object_unref);
+  ZeitgeistEvent *event;
   ZeitgeistTimeRange *time_range = zeitgeist_time_range_new_anytime ();
 
+  if (g_getenv ("ZEITGEIST_FTS_DISABLE_EVENT_BLACKLIST") == NULL)
+  {
+    // Blacklist Ubuntu One events...
+
+    event = zeitgeist_event_new ();
+    zeitgeist_event_set_actor (event, "!dbus://com.ubuntuone.SyncDaemon.service");
+    g_ptr_array_add (templates, event);
+
+    event = zeitgeist_event_new ();
+    zeitgeist_event_set_actor (event, "!dbus://org.desktopcouch.CouchDB.service");
+    g_ptr_array_add (templates, event);
+  }
+
   g_debug ("asking reader for all events");
-  events = zeitgeist_db_reader_find_events (zg_reader,
-                                            time_range,
-                                            templates,
-                                            ZEITGEIST_STORAGE_STATE_ANY,
-                                            0,
-                                            ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                                            NULL,
-                                            &error);
+  event_ids = zeitgeist_db_reader_find_event_ids (zg_reader,
+                                                  time_range,
+                                                  templates,
+                                                  ZEITGEIST_STORAGE_STATE_ANY,
+                                                  0,
+                                                  ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
+                                                  NULL,
+                                                  &event_ids_size,
+                                                  &error);
 
   if (error)
   {
@@ -58,13 +74,18 @@ void Controller::RebuildIndex ()
   }
   else
   {
-    g_debug ("reader returned %u events", events->len);
+    g_debug ("reader returned %d events", event_ids_size);
 
-    IndexEvents (events);
-    g_ptr_array_unref (events);
+    IndexEvents (event_ids, event_ids_size);
+    g_free (event_ids);
 
     // Set the db metadata key only once we're done
     PushTask (new MetadataTask ("fts_index_version", INDEX_VERSION));
+    gint64 zg_creation_date = indexer->GetZeitgeistCreationDate ();
+    gchar *creation = g_strdup_printf ("%" G_GINT64_FORMAT, zg_creation_date);
+    std::string zg_creation_date_str (creation);
+    PushTask (new MetadataTask ("zg_db_creation_date", zg_creation_date_str));
+    g_free (creation);
   }
 
   g_object_unref (time_range);
@@ -81,6 +102,19 @@ void Controller::IndexEvents (GPtrArray *events)
   }
 }
 
+void Controller::IndexEvents (guint *event_ids, int event_ids_size)
+{
+  const int CHUNK_SIZE = 64;
+  // Break down index tasks into suitable chunks
+  for (int i = 0; i < event_ids_size; i += CHUNK_SIZE)
+  {
+    int num_ids = i + CHUNK_SIZE > event_ids_size ? 
+      event_ids_size - i : CHUNK_SIZE;
+    PushTask (new IndexEventsTask (zg_reader,
+        std::vector<guint> (&event_ids[i], &event_ids[i + num_ids])));
+  }
+}
+
 void Controller::DeleteEvents (guint *event_ids, int event_ids_size)
 {
   // FIXME: Should we break the task here as well?
diff --git a/extensions/fts++/controller.h b/extensions/fts++/controller.h
index b62c2918..e6dd0603 100644
--- a/extensions/fts++/controller.h
+++ b/extensions/fts++/controller.h
@@ -49,6 +49,7 @@ public:
   void RebuildIndex ();
 
   void IndexEvents (GPtrArray *events);
+  void IndexEvents (guint *event_ids, int event_ids_size);
   void DeleteEvents (guint *event_ids, int event_ids_size);
 
   void PushTask (Task* task);
diff --git a/extensions/fts++/indexer.cpp b/extensions/fts++/indexer.cpp
index e7d0c6bf..af96eac7 100644
--- a/extensions/fts++/indexer.cpp
+++ b/extensions/fts++/indexer.cpp
@@ -43,6 +43,7 @@ const std::string FILTER_PREFIX_XDG_CATEGORY = "AC";
 const Xapian::valueno VALUE_EVENT_ID = 0;
 const Xapian::valueno VALUE_TIMESTAMP = 1;
 const Xapian::valueno VALUE_URI_HASH = 2;
+const Xapian::valueno VALUE_ORIGIN_HASH = 3;
 
 #define QUERY_PARSER_FLAGS \
   Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_BOOLEAN | \
@@ -66,8 +67,25 @@ void Indexer::Initialize (GError **error)
     {
       gchar *path = g_build_filename (zeitgeist_utils_get_data_path (),
                                       FTS_MAIN_DIR.c_str (), NULL);
-      this->db = new Xapian::WritableDatabase (path,
-                                               Xapian::DB_CREATE_OR_OPEN);
+      try
+      {
+        this->db = new Xapian::WritableDatabase (path,
+                                                 Xapian::DB_CREATE_OR_OPEN);
+      }
+      catch (const Xapian::DatabaseCorruptError &xp_error)
+      {
+        g_message ("Database is corrupt (%s). Overwriting...",
+            xp_error.get_msg ().c_str ());
+        this->db = new Xapian::WritableDatabase (path,
+                                                 Xapian::DB_CREATE_OR_OVERWRITE);
+      }
+      catch (const Xapian::DatabaseOpeningError &xp_error)
+      {
+        g_message ("Database is corrupt (%s). Overwriting...",
+            xp_error.get_msg ().c_str ());
+        this->db = new Xapian::WritableDatabase (path,
+                                                 Xapian::DB_CREATE_OR_OVERWRITE);
+      }
       g_free (path);
     }
 
@@ -128,6 +146,14 @@ void Indexer::Initialize (GError **error)
   }
 }
 
+gint64 Indexer::GetZeitgeistCreationDate ()
+{
+  ZeitgeistSQLiteDatabase *database = zeitgeist_db_reader_get_database (
+      zg_reader);
+  return zeitgeist_sq_lite_database_schema_get_creation_date (
+      database->database);
+}
+
 /**
  * Returns true if and only if the index is good.
  * Otherwise the index should be rebuild.
@@ -146,6 +172,24 @@ bool Indexer::CheckIndex ()
     return false;
   }
 
+  // Get stored Zeitgeist DB creation date
+  gint64 metadata_date;
+  std::string metadata_date_str (db->get_metadata ("zg_db_creation_date"));
+  if (metadata_date_str.empty ())
+    metadata_date = -1;
+  else
+    metadata_date = g_ascii_strtoll (metadata_date_str.c_str (), NULL, 0);
+
+  // In case the Zeitgeist DB is newer than Xapian, we need to re-build.
+  // This may happen if the Zeitgeist DB gets corrupt and is re-created
+  // from scratch.
+  gint64 database_creation_date = GetZeitgeistCreationDate ();
+  if (database_creation_date != metadata_date)
+  {
+    g_message ("Zeitgeist database has been replaced. Doing full rebuild");
+    return false;
+  }
+
   return true;
 }
 
@@ -262,7 +306,10 @@ std::string Indexer::CompileEventFilterQuery (GPtrArray *templates)
     for (unsigned j = 0; j < subjects->len; j++)
     {
       ZeitgeistSubject *subject = (ZeitgeistSubject*) g_ptr_array_index (subjects, j);
+      // For backwards compatibility, we still check URI
       val = zeitgeist_subject_get_uri (subject);
+      if (!val || val[0] == '\0')
+          val = zeitgeist_subject_get_current_uri (subject);
       if (val && val[0] != '\0')
         tmpl.push_back ("zgsu:" + StringUtils::MangleUri (val));
 
@@ -344,7 +391,7 @@ void Indexer::AddDocFilters (ZeitgeistEvent *event, Xapian::Document &doc)
   for (unsigned j = 0; j < subjects->len; j++)
   {
     ZeitgeistSubject *subject = (ZeitgeistSubject*) g_ptr_array_index (subjects, j);
-    val = zeitgeist_subject_get_uri (subject);
+    val = zeitgeist_subject_get_current_uri (subject);
     if (val && val[0] != '\0')
       doc.add_boolean_term (StringUtils::Truncate (FILTER_PREFIX_SUBJECT_URI + StringUtils::MangleUri (val)));
 
@@ -735,6 +782,7 @@ std::string Indexer::CompileQueryString (const gchar *search_string,
   return query_string;
 }
 
+// FIXME: this is missing the Storage State parameter
 GPtrArray* Indexer::Search (const gchar *search,
                             ZeitgeistTimeRange *time_range,
                             GPtrArray *templates,
@@ -759,7 +807,11 @@ GPtrArray* Indexer::Search (const gchar *search,
         result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS ||
         result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_SUBJECTS ||
         result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS ||
-        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS)
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_ORIGIN)
     {
       maxhits = count;
     }
@@ -774,7 +826,9 @@ GPtrArray* Indexer::Search (const gchar *search,
     }
     else
     {
-      enquire->set_sort_by_value (VALUE_TIMESTAMP, true);
+      bool reversed_sort = not
+          zeitgeist_result_type_is_sort_order_asc (result_type);
+      enquire->set_sort_by_value (VALUE_TIMESTAMP, reversed_sort);
     }
 
     if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS ||
@@ -782,7 +836,19 @@ GPtrArray* Indexer::Search (const gchar *search,
         result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS ||
         result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS)
     {
-        enquire->set_collapse_key (VALUE_URI_HASH);
+      enquire->set_collapse_key (VALUE_URI_HASH);
+    }
+    else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_ORIGIN)
+    {
+      enquire->set_collapse_key (VALUE_ORIGIN_HASH);
+    }
+    else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS)
+    {
+      enquire->set_collapse_key (VALUE_EVENT_ID);
     }
 
     Xapian::Query q(query_parser->parse_query (query_string, QUERY_PARSER_FLAGS));
@@ -824,7 +890,6 @@ GPtrArray* Indexer::Search (const gchar *search,
 
       if (event_templates->len > 0)
       {
-        ZeitgeistTimeRange *time_range = zeitgeist_time_range_new_anytime ();
         results = zeitgeist_db_reader_find_events (zg_reader,
                                                    time_range,
                                                    event_templates,
@@ -833,8 +898,6 @@ GPtrArray* Indexer::Search (const gchar *search,
                                                    result_type,
                                                    NULL,
                                                    error);
-
-        g_object_unref (time_range);
       }
       else
       {
@@ -861,6 +924,210 @@ GPtrArray* Indexer::Search (const gchar *search,
   return results;
 }
 
+static guint32*
+find_event_ids_for_combined_template (ZeitgeistDbReader *zg_reader,
+                                      ZeitgeistWhereClause *query_clause, // steals
+                                      GPtrArray *event_templates, // steals
+                                      guint count,
+                                      ZeitgeistResultType result_type,
+                                      gint *event_ids_length,
+                                      GError **error)
+{
+  g_return_val_if_fail (error == NULL || (error && *error == NULL), NULL);
+
+  ZeitgeistWhereClause *uri_where;
+  uri_where = zeitgeist_db_reader_get_where_clause_from_event_templates (
+      zg_reader, event_templates, error);
+  g_ptr_array_unref (event_templates);
+
+  zeitgeist_where_clause_extend (query_clause, uri_where);
+  g_object_unref (G_OBJECT (uri_where));
+
+  guint32 *event_ids;
+  event_ids = zeitgeist_db_reader_find_event_ids_for_clause (zg_reader,
+      query_clause, count, result_type, event_ids_length, error);
+
+  g_object_unref (query_clause);
+
+  return event_ids;
+}
+
+static GPtrArray*
+find_events_for_result_type_and_ids (ZeitgeistDbReader *zg_reader,
+                                     ZeitgeistTimeRange *time_range,
+                                     GPtrArray *templates,
+                                     ZeitgeistStorageState storage_state,
+                                     unsigned count,
+                                     ZeitgeistResultType result_type,
+                                     std::vector<unsigned> const& event_ids,
+                                     std::map<unsigned, gdouble> &relevancy_map,
+                                     GError **error)
+{
+  GPtrArray *results = NULL;
+  results = zeitgeist_db_reader_get_events (zg_reader,
+                                            const_cast<unsigned*>(&event_ids[0]),
+                                            event_ids.size (),
+                                            NULL,
+                                            error);
+
+  if (error && *error) return NULL;
+
+  if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS ||
+      result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS)
+    return results;
+
+  if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS ||
+      result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_SUBJECTS ||
+      result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS ||
+      result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS)
+  {
+    // need to get the uris from the events and do another find_events call
+    GPtrArray *event_templates;
+    event_templates = g_ptr_array_new_with_free_func (g_object_unref);
+    std::map<std::string, unsigned> remapper;
+
+    for (unsigned i = 0; i < results->len; i++)
+    {
+      ZeitgeistEvent* original_event = (ZeitgeistEvent*) results->pdata[i];
+      unsigned event_id = zeitgeist_event_get_id (original_event);
+      GPtrArray *subjects = zeitgeist_event_get_subjects (original_event);
+      if (subjects == NULL) continue;
+      for (unsigned j = 0; j < subjects->len; j++)
+      {
+        const gchar *subj_uri = zeitgeist_subject_get_current_uri (
+                (ZeitgeistSubject*) subjects->pdata[j]);
+        if (subj_uri == NULL) continue;
+        remapper[subj_uri] = event_id;
+        ZeitgeistEvent *event = zeitgeist_event_new ();
+        ZeitgeistSubject *subject = zeitgeist_subject_new ();
+        zeitgeist_subject_set_current_uri (subject, subj_uri);
+        zeitgeist_event_take_subject (event, subject);
+        g_ptr_array_add (event_templates, event);
+      }
+    }
+
+    g_ptr_array_unref (results);
+
+    // construct custom where clause which combines the original template
+    // with the uris we found
+    ZeitgeistWhereClause *where;
+    where = zeitgeist_db_reader_get_where_clause_for_query (zg_reader,
+        time_range, templates, storage_state, error);
+
+    guint32 *real_event_ids;
+    gint real_event_ids_length;
+
+    real_event_ids = find_event_ids_for_combined_template (zg_reader,
+        where, event_templates, count, result_type, &real_event_ids_length,
+        error);
+
+    if (error && *error) return NULL;
+
+    results = zeitgeist_db_reader_get_events (zg_reader,
+                                              real_event_ids,
+                                              real_event_ids_length,
+                                              NULL,
+                                              error);
+
+    g_free (real_event_ids);
+    real_event_ids = NULL;
+
+    if (error && *error) return NULL;
+
+    // the event ids might have changed, we need to update the relevancy_map
+    for (unsigned i = 0; i < results->len; i++)
+    {
+      ZeitgeistEvent* original_event = (ZeitgeistEvent*) results->pdata[i];
+      unsigned event_id = zeitgeist_event_get_id (original_event);
+      GPtrArray *subjects = zeitgeist_event_get_subjects (original_event);
+      if (subjects == NULL) continue;
+      for (unsigned j = 0; j < subjects->len; j++)
+      {
+        const gchar *subj_uri = zeitgeist_subject_get_current_uri (
+                (ZeitgeistSubject*) subjects->pdata[j]);
+        if (subj_uri == NULL) continue;
+        relevancy_map[event_id] = relevancy_map[remapper[subj_uri]];
+      }
+    }
+
+  }
+  else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_ORIGIN ||
+      result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_ORIGIN ||
+      result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_ORIGIN ||
+      result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_ORIGIN)
+  {
+    // need to get the origins from the events and do another find_events call
+    GPtrArray *event_templates;
+    event_templates = g_ptr_array_new_with_free_func (g_object_unref);
+    std::map<std::string, unsigned> remapper;
+
+    for (unsigned i = 0; i < results->len; i++)
+    {
+      ZeitgeistEvent* original_event = (ZeitgeistEvent*) results->pdata[i];
+      unsigned event_id = zeitgeist_event_get_id (original_event);
+      GPtrArray *subjects = zeitgeist_event_get_subjects (original_event);
+      if (subjects == NULL) continue;
+      for (unsigned j = 0; j < subjects->len; j++)
+      {
+        const gchar *subj_origin = zeitgeist_subject_get_origin ((ZeitgeistSubject*) subjects->pdata[j]);
+        if (subj_origin == NULL) continue;
+        remapper[subj_origin] = event_id;
+        ZeitgeistEvent *event = zeitgeist_event_new ();
+        ZeitgeistSubject *subject = zeitgeist_subject_new ();
+        zeitgeist_subject_set_origin (subject, subj_origin);
+        zeitgeist_event_take_subject (event, subject);
+        g_ptr_array_add (event_templates, event);
+      }
+    }
+
+    g_ptr_array_unref (results);
+
+    // construct custom where clause which combines the original template
+    // with the origins we found
+    ZeitgeistWhereClause *where;
+    where = zeitgeist_db_reader_get_where_clause_for_query (zg_reader,
+        time_range, templates, storage_state, error);
+
+    guint32 *real_event_ids;
+    gint real_event_ids_length;
+
+    real_event_ids = find_event_ids_for_combined_template (zg_reader,
+        where, event_templates, count, result_type, &real_event_ids_length,
+        error);
+
+    if (error && *error) return NULL;
+
+    results = zeitgeist_db_reader_get_events (zg_reader,
+                                              real_event_ids,
+                                              real_event_ids_length,
+                                              NULL,
+                                              error);
+
+    if (error && *error) return NULL;
+
+    g_free (real_event_ids);
+    real_event_ids = NULL;
+
+    // the event ids might have changed, we need to update the relevancy_map
+    for (unsigned i = 0; i < results->len; i++)
+    {
+      ZeitgeistEvent* original_event = (ZeitgeistEvent*) results->pdata[i];
+      unsigned event_id = zeitgeist_event_get_id (original_event);
+      GPtrArray *subjects = zeitgeist_event_get_subjects (original_event);
+      if (subjects == NULL) continue;
+      for (unsigned j = 0; j < subjects->len; j++)
+      {
+        const gchar *subj_origin = zeitgeist_subject_get_origin ((ZeitgeistSubject*) subjects->pdata[j]);
+        if (subj_origin == NULL) continue;
+        relevancy_map[event_id] = relevancy_map[remapper[subj_origin]];
+      }
+    }
+
+  }
+
+  return results;
+}
+
 GPtrArray* Indexer::SearchWithRelevancies (const gchar *search,
                                            ZeitgeistTimeRange *time_range,
                                            GPtrArray *templates,
@@ -880,21 +1147,52 @@ GPtrArray* Indexer::SearchWithRelevancies (const gchar *search,
 
     guint maxhits = count;
 
+    if (storage_state != ZEITGEIST_STORAGE_STATE_ANY)
+    {
+      // FIXME: add support for this by grabing (un)available storages
+      // from the storage table and appending them to the query
+      g_set_error_literal (error,
+                           ZEITGEIST_ENGINE_ERROR,
+                           ZEITGEIST_ENGINE_ERROR_INVALID_ARGUMENT,
+                           "Only ANY storage state is supported");
+      return NULL;
+    }
+
+    bool reversed_sort = not
+        zeitgeist_result_type_is_sort_order_asc (result_type);
+
     if (result_type == RELEVANCY_RESULT_TYPE)
     {
       enquire->set_sort_by_relevance ();
     }
-    else
+    else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS)
     {
-      enquire->set_sort_by_value (VALUE_TIMESTAMP, true);
+      enquire->set_sort_by_relevance_then_value (VALUE_TIMESTAMP, reversed_sort);
+      enquire->set_collapse_key (VALUE_EVENT_ID);
     }
-
-    if (storage_state != ZEITGEIST_STORAGE_STATE_ANY)
+    else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_SUBJECTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS)
+    {
+      enquire->set_sort_by_relevance_then_value (VALUE_TIMESTAMP, reversed_sort);
+      enquire->set_collapse_key (VALUE_URI_HASH);
+    }
+    else if (result_type == ZEITGEIST_RESULT_TYPE_MOST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_RECENT_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_MOST_POPULAR_ORIGIN ||
+        result_type == ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_ORIGIN)
+    {
+      enquire->set_sort_by_relevance_then_value (VALUE_TIMESTAMP, reversed_sort);
+      enquire->set_collapse_key (VALUE_ORIGIN_HASH);
+    }
+    else
     {
       g_set_error_literal (error,
                            ZEITGEIST_ENGINE_ERROR,
                            ZEITGEIST_ENGINE_ERROR_INVALID_ARGUMENT,
-                           "Only ANY stogate state is supported");
+                           "Requested result type is not supported");
       return NULL;
     }
 
@@ -926,6 +1224,8 @@ GPtrArray* Indexer::SearchWithRelevancies (const gchar *search,
                                                 NULL,
                                                 error);
 
+      if (error && *error) return NULL;
+
       if (results->len != relevancy_arr.size ())
       {
         g_warning ("Results don't match relevancies!");
@@ -948,22 +1248,56 @@ GPtrArray* Indexer::SearchWithRelevancies (const gchar *search,
     }
     else
     {
-      g_set_error_literal (error,
-                           ZEITGEIST_ENGINE_ERROR,
-                           ZEITGEIST_ENGINE_ERROR_INVALID_ARGUMENT,
-                           "Only RELEVANCY result type is supported");
-      /*
-       * perhaps something like this could be used here?
+      std::vector<unsigned> event_ids;
       std::map<unsigned, gdouble> relevancy_map;
-      foreach (...)
+      Xapian::MSetIterator iter, end;
+      for (iter = hits.begin (), end = hits.end (); iter != end; ++iter)
       {
+        Xapian::Document doc(iter.get_document ());
+        double unserialized =
+          Xapian::sortable_unserialise (doc.get_value (VALUE_EVENT_ID));
+        unsigned event_id = static_cast<unsigned>(unserialized);
+
+        event_ids.push_back (event_id);
+
         double rank = iter.get_percent () / 100.;
         if (rank > relevancy_map[event_id])
         {
           relevancy_map[event_id] = rank;
         }
       }
-      */
+
+      results = find_events_for_result_type_and_ids (zg_reader, time_range,
+                                                     templates, storage_state,
+                                                     count, result_type,
+                                                     event_ids,
+                                                     relevancy_map, error);
+
+      if (error && *error) return NULL;
+
+      if (results == NULL)
+      {
+        results = g_ptr_array_new ();
+        if (relevancies) *relevancies = NULL;
+        if (relevancies_size) *relevancies_size = 0;
+      }
+      else
+      {
+        if (relevancies)
+        {
+          *relevancies = g_new (gdouble, results->len);
+          for (unsigned i = 0; i < results->len; i++)
+          {
+            ZeitgeistEvent *event = (ZeitgeistEvent*) g_ptr_array_index (results, i);
+            (*relevancies)[i] = relevancy_map[zeitgeist_event_get_id (event)];
+          }
+        }
+
+        if (relevancies_size)
+        {
+          *relevancies_size = results->len;
+        }
+      }
     }
 
     if (matches)
@@ -983,11 +1317,37 @@ GPtrArray* Indexer::SearchWithRelevancies (const gchar *search,
   return results;
 }
 
+static void
+get_digest_for_uri (GChecksum *checksum, const gchar *uri,
+                    guint8 *digest, gsize *digest_size)
+{
+  g_checksum_update (checksum, (guchar *) uri, -1);
+  g_checksum_get_digest (checksum, digest, digest_size);
+  g_checksum_reset (checksum);
+  g_assert (digest_size == NULL || *digest_size == HASH_LENGTH);
+}
+
+static bool
+CheckEventBlacklisted (ZeitgeistEvent *event)
+{
+  // Blacklist Ubuntu One events...
+  const gchar *actor;
+  actor = zeitgeist_event_get_actor (event);
+  if (g_strcmp0(actor, "dbus://com.ubuntuone.SyncDaemon.service") == 0)
+    return true;
+  if (g_strcmp0(actor, "dbus://org.desktopcouch.CouchDB.service") == 0)
+    return true;
+
+  return false;
+}
+
 void Indexer::IndexEvent (ZeitgeistEvent *event)
 {
+  if (blacklisting_enabled and CheckEventBlacklisted (event))
+    return;
+
   try
   {
-    // FIXME: we need to special case MOVE_EVENTs
     const gchar *val;
     guint event_id = zeitgeist_event_get_id (event);
     g_return_if_fail (event_id > 0);
@@ -1016,7 +1376,13 @@ void Indexer::IndexEvent (ZeitgeistEvent *event)
       ZeitgeistSubject *subject;
       subject = (ZeitgeistSubject*) g_ptr_array_index (subjects, i);
 
-      val = zeitgeist_subject_get_uri (subject);
+      // We use current_uri (vs. uri) where since we care about real stuff,
+      // not whatever happened some time ago.
+      //
+      // This will most likely still be the same as URI (unless something
+      // triggers a reindexation of the DB), but at least MOVE_EVENTS
+      // will have the updated URI.
+      val = zeitgeist_subject_get_current_uri (subject);
       if (val == NULL || val[0] == '\0') continue;
 
       std::string uri(val);
@@ -1028,19 +1394,28 @@ void Indexer::IndexEvent (ZeitgeistEvent *event)
         return; // ignore this event completely...
       }
 
+      guint8 uri_hash[HASH_LENGTH + 1];
+      gsize hash_size = HASH_LENGTH;
+
       // We need the subject URI so we can use Xapian's collapse key feature
       // for *_SUBJECT grouping. However, to save space, we'll just save a hash.
       // A better option would be using URI's id, but for that we'd need a SQL
       // query that'd be subject to races.
       // FIXME(?): This doesn't work for events with multiple subjects.
-      g_checksum_update (checksum, (guchar *) uri.c_str (), -1);
-      guint8 uri_hash[HASH_LENGTH + 1];
-      gsize hash_size = HASH_LENGTH;
-      g_checksum_get_digest (checksum, uri_hash, &hash_size);
-      g_checksum_reset (checksum);
-      g_assert (hash_size == HASH_LENGTH);
+      get_digest_for_uri (checksum, uri.c_str (), uri_hash, &hash_size);
       doc.add_value (VALUE_URI_HASH, std::string((char *) uri_hash, hash_size));
 
+      size_t colon_pos = uri.find (':');
+      // FIXME: current_origin once we have that
+      val = zeitgeist_subject_get_origin (subject);
+      // make sure the schemas of the URI and origin are the same
+      if (val && colon_pos != std::string::npos && strncmp (uri.c_str (), val, colon_pos+1) == 0)
+      {
+        hash_size = HASH_LENGTH;
+        get_digest_for_uri (checksum, val, uri_hash, &hash_size);
+        doc.add_value (VALUE_ORIGIN_HASH, std::string((char *) uri_hash, hash_size));
+      }
+
       val = zeitgeist_subject_get_text (subject);
       if (val && val[0] != '\0')
       {
diff --git a/extensions/fts++/indexer.h b/extensions/fts++/indexer.h
index 1fbbb57d..cc00fe66 100644
--- a/extensions/fts++/indexer.h
+++ b/extensions/fts++/indexer.h
@@ -29,7 +29,7 @@
 
 namespace ZeitgeistFTS {
 
-const std::string INDEX_VERSION = "2";
+const std::string INDEX_VERSION = "4";
 
 class Indexer
 {
@@ -48,6 +48,7 @@ public:
   {
     const gchar *home_dir = g_get_home_dir ();
     home_dir_path = home_dir != NULL ? home_dir : "/home";
+    blacklisting_enabled = g_getenv ("ZEITGEIST_FTS_DISABLE_EVENT_BLACKLIST") == NULL;
   }
 
   ~Indexer ()
@@ -79,6 +80,7 @@ public:
   void IndexEvent (ZeitgeistEvent *event);
   void DeleteEvent (guint32 event_id);
   void SetDbMetadata (std::string const& key, std::string const& value);
+  gint64 GetZeitgeistCreationDate ();
 
   GPtrArray* Search (const gchar *search,
                      ZeitgeistTimeRange *time_range,
@@ -129,6 +131,7 @@ private:
 
   guint                     clear_failed_id;
   std::string               home_dir_path;
+  bool                      blacklisting_enabled;
 };
 
 }
diff --git a/extensions/fts++/stringutils.cpp b/extensions/fts++/stringutils.cpp
index 6cc5cba8..9c9d2d7d 100644
--- a/extensions/fts++/stringutils.cpp
+++ b/extensions/fts++/stringutils.cpp
@@ -69,9 +69,13 @@ string Truncate (string const& s, unsigned int nbytes)
  */
 string MangleUri (string const& orig)
 {
-  string s(orig);
+  // the input is supposed to be a uri, so no utf8 characters
+  gchar *casefolded = g_ascii_strdown (orig.c_str (), orig.size ());
+
+  string s(casefolded);
+  g_free (casefolded);
   size_t pos = 0;
-  while ((pos = s.find_first_of (": /", pos)) != string::npos)
+  while ((pos = s.find_first_of (": /-.%", pos)) != string::npos)
   {
     s.replace (pos, 1, 1, '_');
     pos++;
diff --git a/extensions/fts++/task.cpp b/extensions/fts++/task.cpp
index 67346ac6..caa17ced 100644
--- a/extensions/fts++/task.cpp
+++ b/extensions/fts++/task.cpp
@@ -23,10 +23,36 @@ namespace ZeitgeistFTS {
 
 void IndexEventsTask::Process (Indexer *indexer)
 {
-  unsigned end_index = MIN (start_index + event_count, events->len);
-  for (unsigned i = start_index; i < end_index; i++)
+  if (events)
   {
-    indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+    unsigned end_index = MIN (start_index + event_count, events->len);
+    for (unsigned i = start_index; i < end_index; i++)
+    {
+      indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+    }
+  }
+  else if (!event_ids.empty ())
+  {
+    GError *error = NULL;
+    GPtrArray *results = zeitgeist_db_reader_get_events (zg_reader,
+                                                         &event_ids[0],
+                                                         event_ids.size (),
+                                                         NULL,
+                                                         &error);
+    if (error)
+    {
+      g_warning ("Unable to get events: %s", error->message);
+      return;
+    }
+    else
+    {
+      for (unsigned i = 0; i < results->len; i++)
+      {
+        indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (results, i));
+      }
+    }
+
+    g_ptr_array_unref (results);
   }
 }
 
diff --git a/extensions/fts++/task.h b/extensions/fts++/task.h
index 3d4b3dd9..630503a6 100644
--- a/extensions/fts++/task.h
+++ b/extensions/fts++/task.h
@@ -49,16 +49,21 @@ public:
 
   IndexEventsTask (GPtrArray *event_arr, unsigned index, unsigned count)
     : events (event_arr), start_index (index), event_count (count) {}
+  
+  IndexEventsTask (ZeitgeistDbReader *reader, std::vector<unsigned> const &ids)
+    : events (NULL), zg_reader (reader), event_ids (ids) {}
 
   virtual ~IndexEventsTask ()
   {
-    g_ptr_array_unref (events);
+    if (events) g_ptr_array_unref (events);
   }
 
 private:
   GPtrArray *events;
   unsigned start_index;
   unsigned event_count;
+  ZeitgeistDbReader *zg_reader;
+  std::vector<unsigned> event_ids;
 };
 
 class DeleteEventsTask : public Task
diff --git a/extensions/fts++/test/test-indexer.cpp b/extensions/fts++/test/test-indexer.cpp
index 02ad70ef..62ca4118 100644
--- a/extensions/fts++/test/test-indexer.cpp
+++ b/extensions/fts++/test/test-indexer.cpp
@@ -1,5 +1,7 @@
 /*
  * Copyright © 2012 Mikkel Kamstrup Erlandsen <mikkel.kamstrup@gmail.com>
+ * Copyright © 2012 Canonical Ltd.
+ *             By Siegfried-A. Gevatter <siegfried.gevatter@collabora.co.uk>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -62,6 +64,30 @@ teardown (Fixture *fix, gconstpointer data)
   g_object_unref (fix->db);
 }
 
+static void
+assert_nth_result_has_id (GPtrArray* results, guint n, guint32 event_id)
+{
+  g_assert_cmpuint (n, <, results->len);
+  ZeitgeistEvent *event = (ZeitgeistEvent*) results->pdata[n];
+  g_assert (event);
+  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+}
+
+// This function only supports events with a single subject,
+// since that's enough for the tests in this file.
+static void
+assert_nth_result_has_text (GPtrArray* results, int n, const char *text)
+{
+  g_assert_cmpuint (n, <, results->len);
+  ZeitgeistEvent *event = (ZeitgeistEvent*) results->pdata[n];
+  g_assert (event);
+  g_assert_cmpint (zeitgeist_event_num_subjects (event), ==, 1);
+  ZeitgeistSubject *subject = (ZeitgeistSubject*)
+    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
+  g_assert (subject);
+  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, text);
+}
+
 static ZeitgeistEvent* create_test_event1 (void)
 {
   ZeitgeistEvent *event = zeitgeist_event_new ();
@@ -131,7 +157,7 @@ static ZeitgeistEvent* create_test_event4 (void)
   zeitgeist_subject_set_interpretation (subject, ZEITGEIST_NFO_PRESENTATION);
   zeitgeist_subject_set_manifestation (subject, ZEITGEIST_NFO_FILE_DATA_OBJECT);
   zeitgeist_subject_set_uri (subject, "file:///home/username/Documents/my_fabulous_presentation.pdf");
-  zeitgeist_subject_set_text (subject, NULL);
+  zeitgeist_subject_set_text (subject, "test texts");
   zeitgeist_subject_set_mimetype (subject, "application/pdf");
 
   zeitgeist_event_set_interpretation (event, ZEITGEIST_ZG_MODIFY_EVENT);
@@ -163,6 +189,96 @@ static ZeitgeistEvent* create_test_event5 (void)
   return event;
 }
 
+static ZeitgeistEvent* create_test_event6 (void)
+{
+  ZeitgeistEvent *event = zeitgeist_event_new ();
+  ZeitgeistSubject *subject = zeitgeist_subject_new ();
+  
+  zeitgeist_subject_set_interpretation (subject, ZEITGEIST_NFO_PRESENTATION);
+  zeitgeist_subject_set_manifestation (subject, ZEITGEIST_NFO_FILE_DATA_OBJECT);
+  zeitgeist_subject_set_uri (subject, "file:///home/username/Documents/CamelCasePresentation.pdf");
+  zeitgeist_subject_set_text (subject, NULL);
+  zeitgeist_subject_set_mimetype (subject, "application/pdf");
+
+  zeitgeist_event_set_interpretation (event, ZEITGEIST_ZG_MODIFY_EVENT);
+  zeitgeist_event_set_manifestation (event, ZEITGEIST_ZG_USER_ACTIVITY);
+  zeitgeist_event_set_actor (event, "application://libreoffice-impress.desktop");
+  zeitgeist_event_add_subject (event, subject);
+
+  g_object_unref (subject);
+  return event;
+}
+
+static ZeitgeistEvent* create_test_event7 (void)
+{
+  ZeitgeistEvent *event = zeitgeist_event_new ();
+  ZeitgeistSubject *subject = zeitgeist_subject_new ();
+  
+  zeitgeist_subject_set_interpretation (subject, ZEITGEIST_NFO_PRESENTATION);
+  zeitgeist_subject_set_manifestation (subject, ZEITGEIST_NFO_FILE_DATA_OBJECT);
+  zeitgeist_subject_set_uri (subject, "file:///home/username/directory-with-dashes/and.dot/%C4%8C%20some-intl/CamelCasePresentation.pdf");
+  zeitgeist_subject_set_text (subject, "some more texts");
+  zeitgeist_subject_set_mimetype (subject, "application/pdf");
+
+  zeitgeist_event_set_interpretation (event, ZEITGEIST_ZG_MODIFY_EVENT);
+  zeitgeist_event_set_manifestation (event, ZEITGEIST_ZG_USER_ACTIVITY);
+  zeitgeist_event_set_actor (event, "application://libreoffice-impress.desktop");
+  zeitgeist_event_add_subject (event, subject);
+
+  g_object_unref (subject);
+  return event;
+}
+
+static ZeitgeistEvent* create_test_event8 (void)
+{
+  ZeitgeistEvent *event = zeitgeist_event_new ();
+  ZeitgeistSubject *subject = zeitgeist_subject_new ();
+  
+  zeitgeist_subject_set_interpretation (subject, ZEITGEIST_NFO_PRESENTATION);
+  zeitgeist_subject_set_manifestation (subject, ZEITGEIST_NFO_FILE_DATA_OBJECT);
+  zeitgeist_subject_set_uri (subject, "file:///home/username/Documents/my_fabulous_presentation.pdf");
+  zeitgeist_subject_set_current_uri (subject, "file:///home/username/Awesome.pdf");
+  zeitgeist_subject_set_text (subject, "some more textt about a presentation or something");
+  zeitgeist_subject_set_mimetype (subject, "application/pdf");
+
+  zeitgeist_event_set_interpretation (event, ZEITGEIST_ZG_MOVE_EVENT);
+  zeitgeist_event_set_manifestation (event, ZEITGEIST_ZG_USER_ACTIVITY);
+  zeitgeist_event_set_actor (event, "application://nautilus.desktop");
+  zeitgeist_event_add_subject (event, subject);
+
+  g_object_unref (subject);
+  return event;
+}
+
+static ZeitgeistEvent* create_test_event_simple (const char *uri, const char *text)
+{
+  ZeitgeistEvent *event = zeitgeist_event_new ();
+  ZeitgeistSubject *subject = zeitgeist_subject_new ();
+  
+  zeitgeist_subject_set_interpretation (subject, ZEITGEIST_NFO_DOCUMENT);
+  zeitgeist_subject_set_manifestation (subject, ZEITGEIST_NFO_FILE_DATA_OBJECT);
+  zeitgeist_subject_set_uri (subject, uri);
+  zeitgeist_subject_set_text (subject, text);
+  zeitgeist_subject_set_mimetype (subject, "text/plain");
+
+  zeitgeist_event_set_interpretation (event, ZEITGEIST_ZG_ACCESS_EVENT);
+  zeitgeist_event_set_manifestation (event, ZEITGEIST_ZG_USER_ACTIVITY);
+  zeitgeist_event_set_actor (event, "application://gedit.desktop");
+  zeitgeist_event_add_subject (event, subject);
+
+  g_object_unref (subject);
+  return event;
+}
+
+static void
+process_pending (Fixture *fix)
+{
+  while (zeitgeist_indexer_has_pending_tasks (fix->indexer))
+  {
+    zeitgeist_indexer_process_task (fix->indexer);
+  }
+}
+
 // Steals the event, ref it if you want to keep it
 static guint
 index_event (Fixture *fix, ZeitgeistEvent *event)
@@ -172,6 +288,7 @@ index_event (Fixture *fix, ZeitgeistEvent *event)
   guint *event_ids;
   int num_events_inserted;
 
+  zeitgeist_event_set_timestamp (event, zeitgeist_timestamp_now ());
   // add event to DBs
   events = g_ptr_array_new ();
   g_ptr_array_add (events, event);
@@ -187,20 +304,73 @@ index_event (Fixture *fix, ZeitgeistEvent *event)
   zeitgeist_indexer_index_events (fix->indexer, events);
   g_ptr_array_unref (events);
 
-  while (zeitgeist_indexer_has_pending_tasks (fix->indexer))
-  {
-    zeitgeist_indexer_process_task (fix->indexer);
-  }
+  process_pending (fix);
+
+  // sleep for 1 msec to make sure the next event will have a
+  // different timestamp
+  g_usleep (1000);
 
   return event_id;
 }
 
+static GPtrArray*
+search_simple (Fixture *fix, const char *text, GPtrArray *templates,
+        ZeitgeistResultType result_type, guint *matches)
+{
+  if (!templates) templates = g_ptr_array_new ();
+  return zeitgeist_indexer_search (fix->indexer,
+                            text,
+                            zeitgeist_time_range_new_anytime (),
+                            templates,
+                            0, // offset
+                            10, // count
+                            result_type,
+                            matches,
+                            NULL);
+}
+
+static GPtrArray*
+search_with_count (Fixture *fix, const char *text, GPtrArray *templates,
+        ZeitgeistResultType result_type, guint offset, guint count,
+        guint *matches)
+{
+  if (!templates) templates = g_ptr_array_new ();
+  return zeitgeist_indexer_search (fix->indexer,
+                            text,
+                            zeitgeist_time_range_new_anytime (),
+                            templates,
+                            offset,
+                            count,
+                            result_type,
+                            matches,
+                            NULL);
+}
+
+static GPtrArray*
+search_with_relevancies_simple (Fixture *fix, const char *text,
+        GPtrArray *templates, ZeitgeistResultType result_type,
+        gdouble **relevancies, gint *relevancies_size, guint *matches)
+{
+  if (!templates) templates = g_ptr_array_new ();
+  return zeitgeist_indexer_search_with_relevancies (fix->indexer,
+                            text,
+                            zeitgeist_time_range_new_anytime (),
+                            templates,
+                            ZEITGEIST_STORAGE_STATE_ANY,
+                            0, // offset
+                            10, // count
+                            result_type,
+                            relevancies,
+                            relevancies_size,
+                            matches,
+                            NULL);
+}
+
 static void
 test_simple_query (Fixture *fix, gconstpointer data)
 {
   guint matches;
   guint event_id;
-  ZeitgeistEvent* event;
  
   // add test events to DBs
   event_id = index_event (fix, create_test_event1 ());
@@ -208,33 +378,61 @@ test_simple_query (Fixture *fix, gconstpointer data)
   index_event (fix, create_test_event3 ());
   index_event (fix, create_test_event4 ());
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              g_ptr_array_new (),
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0, "text");
+}
 
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+static void
+test_simple_query_empty_database (Fixture *fix, gconstpointer data)
+{
+  guint matches;
 
-  ZeitgeistSubject *subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "text");
+  GPtrArray *results = search_simple (fix,
+          "NothingWillEverMatchThisMwhahahaha", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, ==, 0);
+  g_assert_cmpuint (results->len, ==, 0);
+}
+
+static void
+test_simple_query_no_results (Fixture *fix, gconstpointer data)
+{
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  index_event (fix, create_test_event4 ());
+
+  test_simple_query_empty_database (fix, data);
+}
+
+static void
+test_simple_recognize_schemas (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+
+  // add test events to DBs
+  index_event (fix, create_test_event_simple ("file://a.ok", "getme1"));
+  index_event (fix, create_test_event_simple ("ubuntuone://a.bad", "getme2"));
+
+  GPtrArray *results = search_simple (fix, "getme*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_text (results, 0, "getme1");
 }
 
 static void
 test_simple_with_filter (Fixture *fix, gconstpointer data)
 {
   guint matches;
-  guint event_id;
   ZeitgeistEvent* event;
 
   // add test events to DBs
@@ -246,16 +444,8 @@ test_simple_with_filter (Fixture *fix, gconstpointer data)
   zeitgeist_event_set_interpretation (event, ZEITGEIST_NFO_DOCUMENT);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (results->len, ==, 0);
   g_assert_cmpuint (matches, ==, 0);
@@ -280,38 +470,24 @@ test_simple_with_valid_filter (Fixture *fix, gconstpointer data)
   zeitgeist_event_add_subject (event, subject);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
-
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
-
-  subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "text");
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0, "text");
 }
 
 static void
 test_simple_negation (Fixture *fix, gconstpointer data)
 {
   guint matches;
-  guint event_id;
   ZeitgeistEvent* event;
   ZeitgeistSubject *subject;
 
   // add test events to DBs
-  event_id = index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event1 ());
   index_event (fix, create_test_event2 ());
 
   GPtrArray *filters = g_ptr_array_new_with_free_func (g_object_unref);
@@ -321,16 +497,8 @@ test_simple_negation (Fixture *fix, gconstpointer data)
   zeitgeist_event_add_subject (event, subject);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, ==, 0);
   g_assert_cmpuint (results->len, ==, 0);
@@ -340,12 +508,11 @@ static void
 test_simple_noexpand (Fixture *fix, gconstpointer data)
 {
   guint matches;
-  guint event_id;
   ZeitgeistEvent* event;
   ZeitgeistSubject *subject;
 
   // add test events to DBs
-  event_id = index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event1 ());
   index_event (fix, create_test_event2 ());
 
   GPtrArray *filters = g_ptr_array_new_with_free_func (g_object_unref);
@@ -355,16 +522,8 @@ test_simple_noexpand (Fixture *fix, gconstpointer data)
   zeitgeist_event_add_subject (event, subject);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, ==, 0);
   g_assert_cmpuint (results->len, ==, 0);
@@ -389,26 +548,13 @@ test_simple_noexpand_valid (Fixture *fix, gconstpointer data)
   zeitgeist_event_add_subject (event, subject);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "text",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "text", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
-
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
-
-  subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "text");
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0, "text");
 }
 
 static void
@@ -430,26 +576,14 @@ test_simple_url_unescape (Fixture *fix, gconstpointer data)
   zeitgeist_event_add_subject (event, subject);
   g_ptr_array_add (filters, event); // steals ref
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "love",
-                              zeitgeist_time_range_new_anytime (),
-                              filters,
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "love", filters,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
-
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
-
-  subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "Example.com Wiki Page. Kanji is awesome 漢字");
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0,
+          "Example.com Wiki Page. Kanji is awesome 漢字");
 }
 
 static void
@@ -466,22 +600,12 @@ test_simple_underscores (Fixture *fix, gconstpointer data)
   index_event (fix, create_test_event3 ());
   event_id = index_event (fix, create_test_event4 ());
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "fabulo*",
-                              zeitgeist_time_range_new_anytime (),
-                              g_ptr_array_new (),
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "fabulo*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
-
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+  assert_nth_result_has_id (results, 0, event_id);
 }
 
 static void
@@ -499,26 +623,49 @@ test_simple_camelcase (Fixture *fix, gconstpointer data)
   index_event (fix, create_test_event4 ());
   event_id = index_event (fix, create_test_event5 ());
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "signal",
-                              zeitgeist_time_range_new_anytime (),
-                              g_ptr_array_new (),
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "signal", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+}
 
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+static void
+test_simple_dashes_prefix (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id;
+  ZeitgeistEvent* event;
+  ZeitgeistSubject *subject;
+
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  index_event (fix, create_test_event4 ());
+  index_event (fix, create_test_event5 ());
+  index_event (fix, create_test_event6 ());
+  event_id = index_event (fix, create_test_event7 ());
+
+  GPtrArray *event_template = g_ptr_array_new ();
+  event = zeitgeist_event_new ();
+  subject = zeitgeist_subject_new ();
+  zeitgeist_subject_set_uri (subject,
+      "file:///home/username/directory-with-dashes/*");
+  zeitgeist_event_add_subject (event, subject);
+  g_ptr_array_add (event_template, event);
+
+  GPtrArray *results = search_simple (fix, "pdf", event_template,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
 }
 
 static void
-test_simple_cjk (Fixture *fix, gconstpointer data)
+test_simple_dots_prefix (Fixture *fix, gconstpointer data)
 {
   guint matches;
   guint event_id;
@@ -527,28 +674,82 @@ test_simple_cjk (Fixture *fix, gconstpointer data)
 
   // add test events to DBs
   index_event (fix, create_test_event1 ());
-  event_id = index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  index_event (fix, create_test_event4 ());
+  index_event (fix, create_test_event5 ());
+  index_event (fix, create_test_event6 ());
+  event_id = index_event (fix, create_test_event7 ());
+
+  GPtrArray *event_template = g_ptr_array_new ();
+  event = zeitgeist_event_new ();
+  subject = zeitgeist_subject_new ();
+  zeitgeist_subject_set_uri (subject,
+      "file:///home/username/directory-with-dashes/and.dot/*");
+  zeitgeist_event_add_subject (event, subject);
+  g_ptr_array_add (event_template, event);
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "漢*",
-                              zeitgeist_time_range_new_anytime (),
-                              g_ptr_array_new (),
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "pdf", event_template,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+}
 
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+static void
+test_simple_intl_prefix (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id;
+  ZeitgeistEvent* event;
+  ZeitgeistSubject *subject;
 
-  subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "Example.com Wiki Page. Kanji is awesome 漢字");
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  index_event (fix, create_test_event4 ());
+  index_event (fix, create_test_event5 ());
+  index_event (fix, create_test_event6 ());
+  event_id = index_event (fix, create_test_event7 ());
+
+  GPtrArray *event_template = g_ptr_array_new ();
+  event = zeitgeist_event_new ();
+  subject = zeitgeist_subject_new ();
+  zeitgeist_subject_set_uri (subject,
+      "file:///home/username/directory-with-dashes/and.dot/%C4%8C*");
+  zeitgeist_event_add_subject (event, subject);
+  g_ptr_array_add (event_template, event);
+
+  GPtrArray *results = search_simple (fix, "pdf", event_template,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+}
+
+static void
+test_simple_cjk (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id;
+  ZeitgeistEvent* event;
+  ZeitgeistSubject *subject;
+
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  event_id = index_event (fix, create_test_event2 ());
+
+  GPtrArray *results = search_simple (fix, "漢*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0,
+          "Example.com Wiki Page. Kanji is awesome 漢字");
 }
 
 static void
@@ -564,26 +765,355 @@ test_simple_idn_support (Fixture *fix, gconstpointer data)
   index_event (fix, create_test_event2 ());
   event_id = index_event (fix, create_test_event3 ());
 
-  GPtrArray *results =
-    zeitgeist_indexer_search (fix->indexer,
-                              "παράδειγμα",
-                              zeitgeist_time_range_new_anytime (),
-                              g_ptr_array_new (),
-                              0,
-                              10,
-                              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                              &matches,
-                              NULL);
+  GPtrArray *results = search_simple (fix, "παράδειγμα", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
 
   g_assert_cmpuint (matches, >, 0);
   g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0, "IDNwiki");
+}
 
-  event = (ZeitgeistEvent*) results->pdata[0];
-  g_assert_cmpuint (zeitgeist_event_get_id (event), ==, event_id);
+static void
+test_simple_relevancies_query (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id;
+  gdouble *relevancies;
+  gint relevancies_size;
+  ZeitgeistEvent* event;
+ 
+  // add test events to DBs
+  event_id = index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  index_event (fix, create_test_event4 ());
 
-  subject = (ZeitgeistSubject*)
-    g_ptr_array_index (zeitgeist_event_get_subjects (event), 0);
-  g_assert_cmpstr (zeitgeist_subject_get_text (subject), ==, "IDNwiki");
+  GPtrArray *results = search_with_relevancies_simple (fix, "text", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS,
+          &relevancies, &relevancies_size, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  g_assert_cmpint (relevancies_size, ==, 1);
+  g_assert_cmpfloat (relevancies[0], >=, 1.0);
+  assert_nth_result_has_id (results, 0, event_id);
+  assert_nth_result_has_text (results, 0, "text");
+}
+
+static void
+test_simple_relevancies_subject_query (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  gdouble *relevancies;
+  gint relevancies_size;
+  guint event_id4, event_id5, event_id6;
+ 
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event2 ());
+  index_event (fix, create_test_event3 ());
+  event_id4 = index_event (fix, create_test_event4 ());
+  usleep (50000);
+  event_id5 = index_event (fix, create_test_event5 ());
+  usleep (50000);
+  event_id6 = index_event (fix, create_test_event6 ());
+
+  GPtrArray *results = search_with_relevancies_simple (fix, "user*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS,
+          &relevancies, &relevancies_size, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 3);
+  g_assert_cmpint (relevancies_size, ==, 3);
+
+  // we're creating event 6 after 5 and 4, so it has to be more recent (but it seems
+  // that number of terms indexed matters as well, so careful with the relevancies)
+  assert_nth_result_has_id (results, 0, event_id6);
+}
+
+static void
+test_simple_move_event (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id;
+  ZeitgeistEvent* event;
+ 
+  // add test events to DBs
+  index_event (fix, create_test_event1 ());
+  index_event (fix, create_test_event4 ());
+  event_id = index_event (fix, create_test_event8 ());
+
+  GPtrArray *results = search_simple (fix, "awesome", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id);
+}
+
+static void
+test_query_most_recent (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id1, event_id2, event_id3, event_id4;
+  ZeitgeistEvent* event;
+  GPtrArray* results;
+  gdouble *relevancies;
+  gint relevancies_size;
+ 
+  // add test events to DBs
+  event_id1 = index_event (fix, create_test_event1 ());
+  event_id2 = index_event (fix, create_test_event2 ());
+  event_id3 = index_event (fix, create_test_event3 ());
+  event_id4 = index_event (fix, create_test_event4 ());
+
+  for (int i = 0; i < 4; ++i)
+  {
+    if (i == 0)
+    {
+      // Search for MostRecentEvents
+      results = search_simple (fix, "*text*", NULL,
+              ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+    }
+    else if (i == 1)
+    {
+      // Search for MostRecentSubjects
+      results = search_simple (fix, "*text*", NULL,
+              ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS, &matches);
+    }
+    else if (i == 2)
+    {
+      // SearchWithRelevancies for MostRecentEvents
+      GPtrArray *results = search_with_relevancies_simple (fix, "*text*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
+          &relevancies, &relevancies_size, &matches);
+    }
+    else
+    {
+      // SearchWithRelevancies for MostRecentSubjects
+      GPtrArray *results = search_with_relevancies_simple (fix, "*text*", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS,
+          &relevancies, &relevancies_size, &matches);
+    }
+
+    g_assert_cmpuint (matches, >, 0);
+    g_assert_cmpuint (results->len, ==, 2);
+    assert_nth_result_has_id (results, 0, event_id4);
+    assert_nth_result_has_id (results, 1, event_id1);
+  }
+}
+
+static void
+test_query_least_recent (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id1, event_id2, event_id3, event_id4;
+  ZeitgeistEvent* event;
+  GPtrArray* results;
+  gdouble *relevancies;
+  gint relevancies_size;
+ 
+  // add test events to DBs
+  event_id1 = index_event (fix, create_test_event1 ());
+  event_id2 = index_event (fix, create_test_event2 ());
+  event_id3 = index_event (fix, create_test_event3 ());
+  event_id4 = index_event (fix, create_test_event4 ());
+
+  for (int i = 0; i < 4; ++i)
+  {
+    if (i == 0)
+    {
+      // Search for LeastRecentEvents
+      results = search_simple (fix, "*text*", NULL,
+              ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS, &matches);
+    }
+    else if (i == 1)
+    {
+      // Search for LeastRecentSubjects
+      results = search_simple (fix, "*text*", NULL,
+              ZEITGEIST_RESULT_TYPE_LEAST_RECENT_SUBJECTS, &matches);
+    }
+    else if (i == 2)
+    {
+      // SearchWithRelevancies for LeastRecentEvents
+      GPtrArray *results = search_with_relevancies_simple (fix, "*text*", NULL,
+          ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS,
+          &relevancies, &relevancies_size, &matches);
+    }
+    else
+    {
+      // SearchWithRelevancies for LeastRecentSubjects
+      GPtrArray *results = search_with_relevancies_simple (fix, "*text*", NULL,
+          ZEITGEIST_RESULT_TYPE_LEAST_RECENT_SUBJECTS,
+          &relevancies, &relevancies_size, &matches);
+    }
+
+    g_assert_cmpuint (matches, >, 0);
+    g_assert_cmpuint (results->len, ==, 2);
+    assert_nth_result_has_id (results, 0, event_id1);
+    assert_nth_result_has_id (results, 1, event_id4);
+  }
+}
+
+static void
+test_query_sort_order (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id1, event_id2, event_id3, event_id4;
+  ZeitgeistEvent* event;
+  GPtrArray* results;
+ 
+  // add test events to DBs
+  event_id1 = index_event (fix, create_test_event_simple ("file://uri1", "!sort"));
+  event_id2 = index_event (fix, create_test_event_simple ("file://uri2", "+sort"));
+  event_id3 = index_event (fix, create_test_event_simple ("file://uri3", "-sort"));
+
+  // Get the single most recent event
+  results = search_with_count (fix, "sort!", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, 0, 1, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id3);
+  assert_nth_result_has_text (results, 0, "-sort");
+
+  // Get the single least recent event
+  results = search_with_count (fix, "sort!", NULL,
+          ZEITGEIST_RESULT_TYPE_LEAST_RECENT_EVENTS, 0, 1, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 1);
+  assert_nth_result_has_id (results, 0, event_id1);
+  assert_nth_result_has_text (results, 0, "!sort");
+}
+
+static void
+test_query_with_duplicates (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id1, event_id2, event_id3, event_id4;
+  ZeitgeistEvent* event;
+  GPtrArray* results;
+ 
+  // add test events to DBs
+  const char uri1[] = "file:///home/fibonacci/test.py";
+  const char uri2[] = "file:///home/fibonacci/win.txt";
+  event_id1 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id2 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id3 = index_event (fix, create_test_event_simple (uri2, "test"));
+  event_id4 = index_event (fix, create_test_event_simple (uri1, "test"));
+
+  // Search for MostRecentEvents
+  results = search_simple (fix, "test", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 4);
+  assert_nth_result_has_id (results, 0, event_id4);
+  assert_nth_result_has_id (results, 1, event_id3);
+  assert_nth_result_has_id (results, 2, event_id2);
+  assert_nth_result_has_id (results, 3, event_id1);
+
+  // Search for MostRecentSubjects
+  results = search_simple (fix, "test", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_SUBJECTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 2);
+  assert_nth_result_has_id (results, 0, event_id4);
+  assert_nth_result_has_id (results, 1, event_id3);
+
+  // FIXME: these fail
+/*
+  // Search for MostPopularSubjects
+  results = search_simple (fix, "test", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 2);
+  assert_nth_result_has_id (results, 0, event_id4);
+  assert_nth_result_has_id (results, 1, event_id3);
+
+  // Search for LeastPopularSubjects
+  results = search_simple (fix, "test", NULL,
+          ZEITGEIST_RESULT_TYPE_LEAST_POPULAR_SUBJECTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 2);
+  assert_nth_result_has_id (results, 0, event_id3);
+  assert_nth_result_has_id (results, 1, event_id4); // or event_id1 until stuff gets fixed
+*/
+}
+
+static void
+test_query_most_popular_subjects (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  guint event_id1, event_id2, event_id3, event_id4, event_id5,
+        event_id6, event_id7, event_id8, event_id9;
+  ZeitgeistEvent* event;
+  GPtrArray* results;
+ 
+  // add test events to DBs
+  const char uri1[] = "file:///file1.txt";
+  const char uri2[] = "file:///file2.txt";
+  const char uri3[] = "file:///file3.txt";
+  event_id1 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id2 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id3 = index_event (fix, create_test_event_simple (uri2, "test"));
+  event_id4 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id5 = index_event (fix, create_test_event_simple (uri3, "test"));
+  event_id6 = index_event (fix, create_test_event_simple (uri2, "test"));
+  event_id7 = index_event (fix, create_test_event_simple (uri1, "test"));
+  event_id8 = index_event (fix, create_test_event_simple (uri3, "test"));
+  event_id9 = index_event (fix, create_test_event_simple (uri3, "test"));
+
+  // Search for MostPopularSubjects
+  results = search_simple (fix, "test", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_POPULAR_SUBJECTS, &matches);
+
+  g_assert_cmpuint (matches, >, 0);
+  g_assert_cmpuint (results->len, ==, 3);
+  assert_nth_result_has_id (results, 0, event_id7);
+  assert_nth_result_has_id (results, 1, event_id9);
+  assert_nth_result_has_id (results, 2, event_id6);
+}
+
+static void
+test_index_ignore_ubuntu_one (Fixture *fix, gconstpointer data)
+{
+  guint matches;
+  ZeitgeistEvent *event;
+  GPtrArray *results;
+
+  // add test events to DBs
+  index_event (fix, create_test_event_simple ("ubuntuone:uuid", "failme"));
+  event = create_test_event_simple ("file:///nice%20uri", "failme");
+  zeitgeist_event_set_actor (event, "dbus://com.ubuntuone.SyncDaemon.service");
+  index_event (fix, event);
+
+  results = search_simple (fix, "failme", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (results->len, ==, 0);
+
+  // disabling blacklisting
+  g_setenv ("ZEITGEIST_FTS_DISABLE_EVENT_BLACKLIST", "1", true);
+
+  // create a new FTS instance
+  zeitgeist_indexer_free (fix->indexer);
+  GError *error = NULL;
+  fix->indexer = zeitgeist_indexer_new (fix->db, &error);
+  g_assert (error == NULL);
+
+  // wait for it to rebuild the index
+  process_pending (fix);
+
+  results = search_simple (fix, "failme", NULL,
+          ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS, &matches);
+
+  g_assert_cmpuint (results->len, ==, 1); // we still don't want ubuntuone:uuid
 }
 
 G_BEGIN_DECLS
@@ -597,31 +1127,67 @@ static void discard_message (const gchar *domain,
 
 void test_indexer_create_suite (void)
 {
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleQuery", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Query", Fixture, 0,
               setup, test_simple_query, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleWithFilter", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/QueryEmptyDatabase", Fixture, 0,
+              setup, test_simple_query_empty_database, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/QueryNoResults", Fixture, 0,
+              setup, test_simple_query_no_results, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/RecognizeSchemas", Fixture, 0,
+              setup, test_simple_recognize_schemas, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/WithFilter", Fixture, 0,
               setup, test_simple_with_filter, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleWithValidFilter", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/WithValidFilter", Fixture, 0,
               setup, test_simple_with_valid_filter, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleNegation", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Negation", Fixture, 0,
               setup, test_simple_negation, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleNoexpand", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Noexpand", Fixture, 0,
               setup, test_simple_noexpand, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleNoexpandValid", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/NoexpandValid", Fixture, 0,
               setup, test_simple_noexpand_valid, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleUnderscores", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Underscores", Fixture, 0,
               setup, test_simple_underscores, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/SimpleCamelcase", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Camelcase", Fixture, 0,
               setup, test_simple_camelcase, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/URLUnescape", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/PrefixWithDashes", Fixture, 0,
+              setup, test_simple_dashes_prefix, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/PrefixWithDots", Fixture, 0,
+              setup, test_simple_dots_prefix, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/PrefixWithIntlChars", Fixture, 0,
+              setup, test_simple_intl_prefix, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/URLUnescape", Fixture, 0,
               setup, test_simple_url_unescape, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/IDNSupport", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/IDNSupport", Fixture, 0,
               setup, test_simple_idn_support, teardown);
-  g_test_add ("/Zeitgeist/FTS/Indexer/CJK", Fixture, 0,
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/CJK", Fixture, 0,
               setup, test_simple_cjk, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/Relevancies", Fixture, 0,
+              setup, test_simple_relevancies_query, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/RelevanciesSubject", Fixture, 0,
+              setup, test_simple_relevancies_subject_query, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Simple/MoveEvent", Fixture, 0,
+              setup, test_simple_move_event, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Query/MostRecent", Fixture, 0,
+              setup, test_query_most_recent, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Query/LeastRecent", Fixture, 0,
+              setup, test_query_least_recent, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Query/SortOrder", Fixture, 0,
+              setup, test_query_sort_order, teardown);
+  g_test_add ("/Zeitgeist/FTS/Indexer/Query/Duplicates", Fixture, 0,
+              setup, test_query_with_duplicates, teardown);
+  // FIXME: this one doesn't work atm
+  /*
+  g_test_add ("/Zeitgeist/FTS/Indexer/Query/MostPopularSubjects", Fixture, 0,
+              setup, test_query_most_popular_subjects, teardown);
+  */
+  g_test_add ("/Zeitgeist/FTS/Indexer/Index/IgnoreUbuntuOne", Fixture, 0,
+              setup, test_index_ignore_ubuntu_one, teardown);
 
   // get rid of the "rebuilding index..." messages
   g_log_set_handler (NULL, G_LOG_LEVEL_MESSAGE, discard_message, NULL);
+
+  // do not abort on warning()s, eg. when not finding actor information
+  g_log_set_always_fatal (G_LOG_LEVEL_CRITICAL);
 }
 
 G_END_DECLS
diff --git a/extensions/fts++/test/test-stringutils.cpp b/extensions/fts++/test/test-stringutils.cpp
index e7765c42..7bb34fe8 100644
--- a/extensions/fts++/test/test-stringutils.cpp
+++ b/extensions/fts++/test/test-stringutils.cpp
@@ -76,7 +76,7 @@ test_mangle (Fixture *fix, gconstpointer data)
 
   g_assert_cmpstr ("file", ==, StringUtils::MangleUri("file").c_str ());
   g_assert_cmpstr ("file___", ==, StringUtils::MangleUri("file://").c_str ());
-  g_assert_cmpstr ("http___www.zeitgeist-project.com", ==,
+  g_assert_cmpstr ("http___www_zeitgeist_project_com", ==,
       StringUtils::MangleUri("http://www.zeitgeist-project.com").c_str ());
 
   g_assert_cmpstr ("scheme_no_spaces_in_uris", ==,
diff --git a/extensions/fts++/zeitgeist-fts.vala b/extensions/fts++/zeitgeist-fts.vala
index 79bc1905..7800a828 100644
--- a/extensions/fts++/zeitgeist-fts.vala
+++ b/extensions/fts++/zeitgeist-fts.vala
@@ -54,6 +54,7 @@ namespace Zeitgeist
         private static FtsDaemon? instance;
         private static MainLoop mainloop;
         private static bool name_acquired = false;
+        private static bool zeitgeist_up = false;
 
         private DbReader engine;
         private Indexer indexer;
@@ -99,7 +100,7 @@ namespace Zeitgeist
         }
 
         public async void notify_insert (Variant time_range, Variant events)
-            throws IOError
+            throws IOError, EngineError
         {
             debug ("got insertion notification");
             var events_arr = Events.from_variant (events);
@@ -170,6 +171,16 @@ namespace Zeitgeist
             }
         }
 
+        private static void zeitgeist_vanished ()
+        {
+            if (zeitgeist_up)
+            {
+                // client APIs query us via zeitgeist, so quit if ZG goes away
+                mainloop.quit ();
+            }
+            zeitgeist_up = false;
+        }
+
         static void run ()
             throws Error
         {
@@ -177,10 +188,7 @@ namespace Zeitgeist
             var proxy = connection.get_proxy_sync<RemoteDBus> (
                 "org.freedesktop.DBus", "/org/freedesktop/DBus",
                 DBusProxyFlags.DO_NOT_LOAD_PROPERTIES);
-            bool zeitgeist_up = proxy.name_has_owner (ZEITGEIST_DBUS_NAME);
-            // FIXME: throw an error that zeitgeist isn't up? or just start it?
-            bool name_owned = proxy.name_has_owner (DBUS_NAME);
-            if (name_owned)
+            if (proxy.name_has_owner (DBUS_NAME))
             {
                 throw new EngineError.EXISTING_INSTANCE (
                     "The FTS daemon is running already.");
@@ -214,6 +222,10 @@ namespace Zeitgeist
                 name_acquired_callback,
                 name_lost_callback);
 
+            Bus.watch_name (BusType.SESSION, ZEITGEIST_DBUS_NAME, 0,
+                () => { zeitgeist_up = true; },
+                zeitgeist_vanished);
+
             mainloop = new MainLoop ();
             mainloop.run ();
 
diff --git a/extensions/fts.vala b/extensions/fts.vala
index 1bc1563f..53312033 100644
--- a/extensions/fts.vala
+++ b/extensions/fts.vala
@@ -95,24 +95,22 @@ namespace Zeitgeist
 
                 // FIXME: shouldn't we delay this to next idle callback?
                 // Get SimpleIndexer
-                Bus.watch_name_on_connection (connection,
+                connection.get_proxy.begin<RemoteSimpleIndexer> (
                     INDEXER_NAME,
-                    BusNameWatcherFlags.AUTO_START,
-                    (conn) =>
-                    {
-                        if (siin != null) return;
-                        conn.get_proxy.begin<RemoteSimpleIndexer> (
-                            "org.gnome.zeitgeist.SimpleIndexer",
-                            "/org/gnome/zeitgeist/index/activity",
-                            0, null, this.proxy_acquired);
-                    },
-                    () => {});
+                    "/org/gnome/zeitgeist/index/activity",
+                    0, null, this.proxy_acquired);
             }
             catch (Error err)
             {
                 warning ("%s", err.message);
             }
         }
+        
+        private void proxy_not_present()
+        {
+            notifier.remove_monitor (new BusName (INDEXER_NAME),"/org/gnome/zeitgeist/monitor/special");
+            this.unload();
+        }
 
         private void proxy_acquired (Object? obj, AsyncResult res)
         {
@@ -120,7 +118,15 @@ namespace Zeitgeist
             try
             {
                 siin = conn.get_proxy.end<RemoteSimpleIndexer> (res);
-                siin_connection_failed = false;
+                if((siin as DBusProxy).g_name_owner == null)
+                {
+                    this.proxy_not_present();
+                    siin_connection_failed = true;
+                }
+                else
+                {
+                    siin_connection_failed = false;
+                }
             }
             catch (IOError err)
             {
@@ -147,6 +153,25 @@ namespace Zeitgeist
                     "Not connected to SimpleIndexer");
             }
         }
+        
+        public override void unload ()
+        {
+            try
+            {
+                var connection = Bus.get_sync (BusType.SESSION, null);
+                if (registration_id != 0)
+                {
+                    connection.unregister_object (registration_id);
+                    registration_id = 0;
+                }
+            }
+            catch (Error err)
+            {
+                warning ("%s", err.message);
+            }
+
+            debug ("%s, this.ref_count = %u", Log.METHOD, this.ref_count);
+        }
 
         public async void search (string query_string, Variant time_range,
             Variant filter_templates, uint offset, uint count, uint result_type,
diff --git a/extensions/histogram.vala b/extensions/histogram.vala
index b5f02f0e..d5aa28a1 100644
--- a/extensions/histogram.vala
+++ b/extensions/histogram.vala
@@ -96,14 +96,8 @@ namespace Zeitgeist
 
                 builder.add ("(xu)", t, count);
             }
-
-            if (rc != Sqlite.DONE)
-            {
-                string error_message = "Error in get_histogram_data: " +
-                    "%d, %s".printf (rc, db.errmsg ());
-                warning ("%s", error_message);
-                throw new EngineError.DATABASE_ERROR (error_message);
-            }
+            database.assert_query_success (rc, "Error in get_histogram_data",
+                Sqlite.DONE);
 
             return builder.end ();
         }
diff --git a/extensions/storage-monitor.vala b/extensions/storage-monitor.vala
index 7f6256bf..a0010e4a 100644
--- a/extensions/storage-monitor.vala
+++ b/extensions/storage-monitor.vala
@@ -112,6 +112,7 @@ namespace Zeitgeist
 
         private Sqlite.Statement get_storages_stmt;
         private Sqlite.Statement store_storage_medium_stmt;
+        private Sqlite.Statement update_storage_medium_stmt;
         private Sqlite.Statement insert_unavailable_medium_stmt;
         private Sqlite.Statement update_medium_state_stmt;
 
@@ -236,7 +237,7 @@ namespace Zeitgeist
             database.assert_query_success (rc, "Storage retrieval query error");
 
             sql = """
-                INSERT OR REPLACE INTO storage (
+                INSERT INTO storage (
                     value, state, icon, display_name
                 ) VALUES (
                     ?, ?, ?, ?
@@ -245,6 +246,14 @@ namespace Zeitgeist
             database.assert_query_success (rc, "Storage insertion query error");
 
             sql = """
+                UPDATE storage SET 
+                state=?, icon=?, display_name=? 
+                WHERE value=?
+                """;
+            rc = db.prepare_v2 (sql, -1, out update_storage_medium_stmt);
+            database.assert_query_success (rc, "Storage update query error");
+
+            sql = """
                 INSERT INTO storage (
                     state, value
                 ) VALUES (
@@ -309,6 +318,8 @@ namespace Zeitgeist
             return "unknown";
         }
 
+        /*
+        // It is not being used since gvfs is not being friendly
         private void on_volume_added (Volume volume)
         {
             debug ("volume added");
@@ -320,19 +331,21 @@ namespace Zeitgeist
             add_storage_medium (get_volume_id (volume), icon_name,
                 volume.get_name ());
         }
-
+        
         private void on_volume_removed (Volume volume)
         {
             debug ("Volume removed");
             remove_storage_medium (get_volume_id (volume));
         }
+        */
 
         /*
          * Return a string identifier for a GIO Volume. This id is constructed
          * as a `best effort` since we can not always uniquely identify
          * volumes, especially audio- and data CDs are problematic.
          */
-        private string get_volume_id (Volume volume)
+        
+        /*private string get_volume_id (Volume volume)
         {
             string volume_id;
 
@@ -349,7 +362,7 @@ namespace Zeitgeist
                 return volume_id;
 
             return "unknown";
-        }
+        }*/
 
         public void add_storage_medium (string medium_name, string icon,
             string display_name)
@@ -360,11 +373,23 @@ namespace Zeitgeist
             store_storage_medium_stmt.bind_int (2, 1);
             store_storage_medium_stmt.bind_text (3, icon);
             store_storage_medium_stmt.bind_text (4, display_name);
-
-            int rc = store_storage_medium_stmt.step ();
-            database.assert_query_success (rc, "add_storage_medium",
-                Sqlite.DONE);
-
+            if (store_storage_medium_stmt.step () != Sqlite.DONE)
+            {
+                update_storage_medium_stmt.reset ();
+                update_storage_medium_stmt.bind_int (1, 1);
+                update_storage_medium_stmt.bind_text (2, icon);
+                update_storage_medium_stmt.bind_text (3, display_name);
+                update_storage_medium_stmt.bind_text (4, medium_name);
+                int rc = update_storage_medium_stmt.step ();
+                try
+                {
+                    database.assert_query_success (rc, "add_storage_medium", Sqlite.DONE);
+                }
+                catch (EngineError e)
+                {
+                    warning ("Could not add storage medium: %s", e.message);
+                }
+            }
             storage_available (medium_name, StorageMedia.to_variant (
                 medium_name, true, icon, display_name));
         }
@@ -381,8 +406,15 @@ namespace Zeitgeist
                 update_medium_state_stmt.bind_int (1, 0);
                 update_medium_state_stmt.bind_text (2, medium_name);
                 int rc = update_medium_state_stmt.step ();
-                database.assert_query_success (rc, "remove_storage_medium",
-                    Sqlite.DONE);
+                try
+                {
+                    database.assert_query_success (rc, "remove_storage_medium",
+                        Sqlite.DONE);
+                }
+                catch (EngineError e)
+                {
+                    warning ("Could not remove storage medium: %s", e.message);
+                }
             }
             storage_unavailable (medium_name);
         }
diff --git a/python/mimetypes.py b/python/mimetypes.py
index 482dabfd..a4e80572 100644
--- a/python/mimetypes.py
+++ b/python/mimetypes.py
@@ -103,6 +103,7 @@ MIMES = {
 
     # Plain text
     "text/plain": Interpretation.TEXT_DOCUMENT,
+    "text/csv": Interpretation.TEXT_DOCUMENT,
   
     # HTML files on disk are always HTML_DOCUMENTS while online we should
     # assume them to be WEBSITEs. By default we anticipate local files...
@@ -111,11 +112,13 @@ MIMES = {
     # Image types
     "application/vnd.corel-draw": Interpretation.VECTOR_IMAGE,
     "image/jpeg": Interpretation.RASTER_IMAGE,
+    "image/pjpeg": Interpretation.RASTER_IMAGE,
     "image/png": Interpretation.RASTER_IMAGE,
     "image/tiff": Interpretation.RASTER_IMAGE,
     "image/gif": Interpretation.RASTER_IMAGE,
     "image/x-xcf": Interpretation.RASTER_IMAGE,
     "image/svg+xml": Interpretation.VECTOR_IMAGE,
+    "image/vnd.microsoft.icon": Interpretation.ICON,
     
     # Audio
     "application/ogg": Interpretation.AUDIO,
@@ -124,6 +127,9 @@ MIMES = {
     # Development files
     "application/ecmascript": Interpretation.SOURCE_CODE,
     "application/javascript": Interpretation.SOURCE_CODE,
+    "application/json": Interpretation.SOURCE_CODE,
+    "application/soap+xml": Interpretation.SOURCE_CODE,
+    "application/xml-dtd": Interpretation.SOURCE_CODE,
     "application/x-csh": Interpretation.SOURCE_CODE,
     "application/x-designer": Interpretation.SOURCE_CODE,
     "application/x-dia-diagram": Interpretation.SOURCE_CODE,
@@ -131,15 +137,17 @@ MIMES = {
     "application/x-glade": Interpretation.SOURCE_CODE,
     "application/xhtml+xml": Interpretation.SOURCE_CODE,
     "application/x-java-archive": Interpretation.SOURCE_CODE,
+    "application/x-javascript": Interpretation.SOURCE_CODE,
     "application/x-m4": Interpretation.SOURCE_CODE,
     "application/xml": Interpretation.SOURCE_CODE,
-    "application/x-object": Interpretation.SOURCE_CODE,
     "application/x-perl": Interpretation.SOURCE_CODE,
     "application/x-php": Interpretation.SOURCE_CODE,
     "application/x-ruby": Interpretation.SOURCE_CODE,
     "application/x-shellscript": Interpretation.SOURCE_CODE,
     "application/x-sql": Interpretation.SOURCE_CODE,
     "text/css": Interpretation.SOURCE_CODE,
+    "text/javascript": Interpretation.SOURCE_CODE,
+    "text/xml": Interpretation.SOURCE_CODE,
     "text/x-c": Interpretation.SOURCE_CODE,
     "text/x-c++": Interpretation.SOURCE_CODE,
     "text/x-chdr": Interpretation.SOURCE_CODE,
@@ -169,6 +177,15 @@ MIMES = {
     "text/x-vala": Interpretation.SOURCE_CODE,
     "text/x-vhdl": Interpretation.SOURCE_CODE,
     "text/x-m4": Interpretation.SOURCE_CODE,
+    "text/x-jquery-tmpl": Interpretation.SOURCE_CODE,
+    
+    # Email
+    "message/alternative": Interpretation.EMAIL,
+    "message/partial": Interpretation.EMAIL,
+    "message/related": Interpretation.EMAIL,
+    
+    # People
+    "text/vcard": Interpretation.CONTACT,
     
     # Archives
     "application/zip": Interpretation.ARCHIVE,
@@ -180,6 +197,7 @@ MIMES = {
     "application/x-bzip-compressed-tar": Interpretation.ARCHIVE,
     "application/x-lzma-compressed-tar": Interpretation.ARCHIVE,
     "application/x-compressed-tar": Interpretation.ARCHIVE,
+    "application/x-stuffit": Interpretation.ARCHIVE,
     
     # Software and packages
     "application/x-deb": Interpretation.SOFTWARE,
@@ -187,6 +205,7 @@ MIMES = {
     "application/x-ms-dos-executable": Interpretation.SOFTWARE,
     "application/x-executable": Interpretation.SOFTWARE,
     "application/x-desktop": Interpretation.SOFTWARE,
+    "application/x-shockwave-flash": Interpretation.EXECUTABLE,
     
     # File systems
     "application/x-cd-image": Interpretation.FILESYSTEM_IMAGE,
@@ -206,6 +225,9 @@ MIMES_REGEX = make_regex_tuple(
     # MS
     ("application/vnd.ms-excel.*", Interpretation.SPREADSHEET),
     ("application/vnd.ms-powerpoint.*", Interpretation.PRESENTATION),
+    ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.*", Interpretation.SPREADSHEET),
+    ("application/vnd.openxmlformats-officedocument.presentationml.presentation.*", Interpretation.PRESENTATION),
+    ("application/vnd.openxmlformats-officedocument.wordprocessingml.document.*", Interpretation.PAGINATED_TEXT_DOCUMENT),
     # TeX stuff
     (".*/x-dvi", Interpretation.PAGINATED_TEXT_DOCUMENT),
     # Image types
diff --git a/src/datamodel.vala b/src/datamodel.vala
index c81e2101..844baf5f 100644
--- a/src/datamodel.vala
+++ b/src/datamodel.vala
@@ -76,7 +76,7 @@ namespace Zeitgeist
         }
 
         public TimeRange.from_variant (Variant variant)
-            throws EngineError.INVALID_SIGNATURE
+            throws EngineError
         {
             assert_sig (variant.get_type_string () == "(xx)",
                 "Invalid D-Bus signature.");
@@ -230,10 +230,60 @@ namespace Zeitgeist
                                                    // different origin ordered
                                                    // by the popularity of the
                                                    // origins
-        LEAST_POPULAR_EVENT_ORIGIN           = 30, //   The last event of each
+        LEAST_POPULAR_EVENT_ORIGIN           = 30; //   The last event of each
                                                    // different origin, ordered
                                                    // ascendingly by the
                                                    // popularity of the origin
+
+        /*
+         * Returns true if the results for the given result_type will be sorted
+         * ascendantly by date, false if they'll be sorted descendingly.
+         **/
+        public static bool is_sort_order_asc (ResultType result_type)
+        {
+            switch (result_type)
+            {
+                // FIXME: Why are LEAST_POPULAR_* using ASC?
+                case ResultType.LEAST_RECENT_EVENTS:
+                case ResultType.LEAST_RECENT_EVENT_ORIGIN:
+                case ResultType.LEAST_POPULAR_EVENT_ORIGIN:
+                case ResultType.LEAST_RECENT_SUBJECTS:
+                case ResultType.LEAST_POPULAR_SUBJECTS:
+                case ResultType.LEAST_RECENT_CURRENT_URI:
+                case ResultType.LEAST_POPULAR_CURRENT_URI:
+                case ResultType.LEAST_RECENT_ACTOR:
+                case ResultType.LEAST_POPULAR_ACTOR:
+                case ResultType.OLDEST_ACTOR:
+                case ResultType.LEAST_RECENT_ORIGIN:
+                case ResultType.LEAST_POPULAR_ORIGIN:
+                case ResultType.LEAST_RECENT_SUBJECT_INTERPRETATION:
+                case ResultType.LEAST_POPULAR_SUBJECT_INTERPRETATION:
+                case ResultType.LEAST_RECENT_MIMETYPE:
+                case ResultType.LEAST_POPULAR_MIMETYPE:
+                    return true;
+
+                case ResultType.MOST_RECENT_EVENTS:
+                case ResultType.MOST_RECENT_EVENT_ORIGIN:
+                case ResultType.MOST_POPULAR_EVENT_ORIGIN:
+                case ResultType.MOST_RECENT_SUBJECTS:
+                case ResultType.MOST_POPULAR_SUBJECTS:
+                case ResultType.MOST_RECENT_CURRENT_URI:
+                case ResultType.MOST_POPULAR_CURRENT_URI:
+                case ResultType.MOST_RECENT_ACTOR:
+                case ResultType.MOST_POPULAR_ACTOR:
+                case ResultType.MOST_RECENT_ORIGIN:
+                case ResultType.MOST_POPULAR_ORIGIN:
+                case ResultType.MOST_RECENT_SUBJECT_INTERPRETATION:
+                case ResultType.MOST_POPULAR_SUBJECT_INTERPRETATION:
+                case ResultType.MOST_RECENT_MIMETYPE:
+                case ResultType.MOST_POPULAR_MIMETYPE:
+                    return false;
+
+                default:
+                    warning ("Unrecognized ResultType: %u", (uint) result_type);
+                    return true;
+            }
+        }
     }
 
     /*
@@ -348,6 +398,11 @@ namespace Zeitgeist
             subjects.add (subject);
         }
 
+        public void take_subject (owned Subject subject)
+        {
+            subjects.add ((owned) subject);
+        }
+
         public Event.full (string? interpretation=null,
             string? manifestation=null, string? actor=null,
             string? origin=null, ...)
@@ -566,6 +621,46 @@ namespace Zeitgeist
             return vb.end ();
         }
 
+        /* Same as to_variant but raises an exception if the variant size
+         * exceeds `limit' bytes.
+         * */
+        public static Variant to_variant_with_limit (GenericArray<Event?> events,
+            size_t limit=Utils.MAX_DBUS_RESULT_SIZE) throws EngineError
+        {
+            var vb = new VariantBuilder(new VariantType("a("+Utils.SIG_EVENT+")"));
+
+            size_t variant_size = 0;
+
+            for (int i = 0; i < events.length; ++i)
+            {
+                Variant event_variant;
+
+                if (events[i] != null)
+                {
+                    event_variant = events[i].to_variant ();
+                }
+                else
+                {
+                    event_variant = get_null_event_variant ();
+                }
+
+                variant_size += event_variant.get_size();
+                if (variant_size > limit)
+                {
+                    size_t avg_event_size = variant_size / (i+1);
+                    string error_message = ("Query exceeded size limit of % " +
+                        size_t.FORMAT + "MiB (roughly ~%d events).").printf (
+                            limit / 1024 / 1024, limit / avg_event_size);
+                    warning (error_message);
+                    throw new EngineError.TOO_MANY_RESULTS (error_message);
+                }
+
+                vb.add_value (event_variant);
+            }
+
+            return vb.end ();
+        }
+
         private static Variant get_null_event_variant ()
         {
             var vb = new VariantBuilder (new VariantType ("("+Utils.SIG_EVENT+")"));
diff --git a/src/db-reader.vala b/src/db-reader.vala
index d4f27f96..e1626459 100644
--- a/src/db-reader.vala
+++ b/src/db-reader.vala
@@ -57,15 +57,23 @@ public class DbReader : Object
         database.set_deletion_callback (delete_from_cache);
         db = database.database;
 
-        interpretations_table = new TableLookup (database, "interpretation");
-        manifestations_table = new TableLookup (database, "manifestation");
-        mimetypes_table = new TableLookup (database, "mimetype");
-        actors_table = new TableLookup (database, "actor");
-
         cache = new EventCache ();
+        try
+        {
+            interpretations_table = new TableLookup (database, "interpretation");
+            manifestations_table = new TableLookup (database, "manifestation");
+            mimetypes_table = new TableLookup (database, "mimetype");
+            actors_table = new TableLookup (database, "actor");
+        }
+        catch (EngineError err)
+        {
+            // FIXME: propagate this properly?
+            critical ("TableLookup initialization failed: %s", err.message);
+        }
     }
 
     protected Event get_event_from_row (Sqlite.Statement stmt, uint32 event_id)
+        throws EngineError
     {
         Event event = new Event ();
         event.id = event_id;
@@ -92,6 +100,7 @@ public class DbReader : Object
     }
 
     protected Subject get_subject_from_row (Sqlite.Statement stmt)
+        throws EngineError
     {
         Subject subject = new Subject ();
         subject.uri = stmt.column_text (EventViewRows.SUBJECT_URI);
@@ -162,11 +171,7 @@ public class DbReader : Object
             Subject subject = get_subject_from_row(stmt);
             event.add_subject(subject);
         }
-        if (rc != Sqlite.DONE)
-        {
-            throw new EngineError.DATABASE_ERROR ("Error: %d, %s\n",
-                rc, db.errmsg ());
-        }
+        database.assert_query_success (rc, "Error", Sqlite.DONE);
 
         // Sort events according to the sequence of event_ids
         results.length = event_ids.length;
@@ -182,47 +187,9 @@ public class DbReader : Object
         return results;
     }
 
-    public uint32[] find_event_ids (TimeRange time_range,
-        GenericArray<Event> event_templates,
-        uint storage_state, uint max_events, uint result_type,
-        BusName? sender=null) throws EngineError
+    public uint32[] find_event_ids_for_clause (WhereClause where,
+        uint max_events, uint result_type) throws EngineError
     {
-
-        WhereClause where = new WhereClause (WhereClause.Type.AND);
-
-        /**
-         * We are using the unary operator here to tell SQLite to not use
-         * the index on the timestamp column at the first place. This is a
-         * "fix" for (LP: #672965) based on some benchmarks, which suggest
-         * a performance win, but we might not oversee all implications.
-         * (See http://www.sqlite.org/optoverview.html, section 6.0).
-         *    -- Markus Korn, 29/11/2010
-         */
-        if (time_range.start != 0)
-            where.add (("+timestamp >= %" + int64.FORMAT).printf(
-                time_range.start));
-        if (time_range.end != 0)
-            where.add (("+timestamp <= %" + int64.FORMAT).printf(
-                time_range.end));
-
-        if (storage_state == StorageState.AVAILABLE ||
-            storage_state == StorageState.NOT_AVAILABLE)
-        {
-            where.add ("(subj_storage_state=? OR subj_storage_state IS NULL)",
-                storage_state.to_string ());
-        }
-        else if (storage_state != StorageState.ANY)
-        {
-            throw new EngineError.INVALID_ARGUMENT(
-                "Unknown storage state '%u'".printf(storage_state));
-        }
-
-        WhereClause tpl_conditions = get_where_clause_from_event_templates (
-            event_templates);
-        where.extend (tpl_conditions);
-        //if (!where.may_have_results ())
-        //    return new uint32[0];
-
         string sql = "SELECT id FROM event_view ";
         string where_sql = "";
         if (!where.is_empty ())
@@ -233,103 +200,97 @@ public class DbReader : Object
         switch (result_type)
         {
             case ResultType.MOST_RECENT_EVENTS:
-                sql += where_sql + " ORDER BY timestamp DESC";
+                sql += where_sql + " ORDER BY ";
                 break;
             case ResultType.LEAST_RECENT_EVENTS:
-                sql += where_sql + " ORDER BY timestamp ASC";
+                sql += where_sql + " ORDER BY ";
                 break;
             case ResultType.MOST_RECENT_EVENT_ORIGIN:
-                sql += group_and_sort ("origin", where_sql, false);
+                sql += group_and_sort ("origin", where_sql);
                 break;
             case ResultType.LEAST_RECENT_EVENT_ORIGIN:
-                sql += group_and_sort ("origin", where_sql, true);
+                sql += group_and_sort ("origin", where_sql);
                 break;
             case ResultType.MOST_POPULAR_EVENT_ORIGIN:
-                sql += group_and_sort ("origin", where_sql, false, false);
+                sql += group_and_sort ("origin", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_EVENT_ORIGIN:
-                sql += group_and_sort ("origin", where_sql, true, true);
+                sql += group_and_sort ("origin", where_sql, true);
                 break;
             case ResultType.MOST_RECENT_SUBJECTS:
-                sql += group_and_sort ("subj_id", where_sql, false);
+                sql += group_and_sort ("subj_id", where_sql);
                 break;
             case ResultType.LEAST_RECENT_SUBJECTS:
-                sql += group_and_sort ("subj_id", where_sql, true);
+                sql += group_and_sort ("subj_id", where_sql);
                 break;
             case ResultType.MOST_POPULAR_SUBJECTS:
-                sql += group_and_sort ("subj_id", where_sql, false, false);
+                sql += group_and_sort ("subj_id", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_SUBJECTS:
-                sql += group_and_sort ("subj_id", where_sql, true, true);
+                sql += group_and_sort ("subj_id", where_sql, true);
                 break;
             case ResultType.MOST_RECENT_CURRENT_URI:
-                sql += group_and_sort ("subj_id_current", where_sql, false);
+                sql += group_and_sort ("subj_id_current", where_sql);
                 break;
             case ResultType.LEAST_RECENT_CURRENT_URI:
-                sql += group_and_sort ("subj_id_current", where_sql, true);
+                sql += group_and_sort ("subj_id_current", where_sql);
                 break;
             case ResultType.MOST_POPULAR_CURRENT_URI:
-                sql += group_and_sort ("subj_id_current", where_sql,
-                    false, false);
+                sql += group_and_sort ("subj_id_current", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_CURRENT_URI:
-                sql += group_and_sort ("subj_id_current", where_sql,
-                    true, true);
+                sql += group_and_sort ("subj_id_current", where_sql, true);
                 break;
             case ResultType.MOST_RECENT_ACTOR:
-                sql += group_and_sort ("actor", where_sql, false);
+                sql += group_and_sort ("actor", where_sql);
                 break;
             case ResultType.LEAST_RECENT_ACTOR:
-                sql += group_and_sort ("actor", where_sql, true);
+                sql += group_and_sort ("actor", where_sql);
                 break;
             case ResultType.MOST_POPULAR_ACTOR:
-                sql += group_and_sort ("actor", where_sql, false, false);
+                sql += group_and_sort ("actor", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_ACTOR:
-                sql += group_and_sort ("actor", where_sql, true, true);
+                sql += group_and_sort ("actor", where_sql, true);
                 break;
             case ResultType.OLDEST_ACTOR:
-                sql += group_and_sort ("actor", where_sql, true, null, "min");
+                sql += group_and_sort ("actor", where_sql, null, "min");
                 break;
             case ResultType.MOST_RECENT_ORIGIN:
-                sql += group_and_sort ("subj_origin", where_sql, false);
+                sql += group_and_sort ("subj_origin", where_sql);
                 break;
             case ResultType.LEAST_RECENT_ORIGIN:
-                sql += group_and_sort ("subj_origin", where_sql, true);
+                sql += group_and_sort ("subj_origin", where_sql);
                 break;
             case ResultType.MOST_POPULAR_ORIGIN:
-                sql += group_and_sort ("subj_origin", where_sql, false, false);
+                sql += group_and_sort ("subj_origin", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_ORIGIN:
-                sql += group_and_sort ("subj_origin", where_sql, true, true);
+                sql += group_and_sort ("subj_origin", where_sql, true);
                 break;
             case ResultType.MOST_RECENT_SUBJECT_INTERPRETATION:
-                sql += group_and_sort ("subj_interpretation", where_sql, false);
+                sql += group_and_sort ("subj_interpretation", where_sql);
                 break;
             case ResultType.LEAST_RECENT_SUBJECT_INTERPRETATION:
-                sql += group_and_sort ("subj_interpretation", where_sql, true);
+                sql += group_and_sort ("subj_interpretation", where_sql);
                 break;
             case ResultType.MOST_POPULAR_SUBJECT_INTERPRETATION:
-                sql += group_and_sort ("subj_interpretation", where_sql,
-                    false, false);
+                sql += group_and_sort ("subj_interpretation", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_SUBJECT_INTERPRETATION:
-                sql += group_and_sort ("subj_interpretation", where_sql,
-                    true, true);
+                sql += group_and_sort ("subj_interpretation", where_sql, true);
                 break;
             case ResultType.MOST_RECENT_MIMETYPE:
-                sql += group_and_sort ("subj_mimetype", where_sql, false);
+                sql += group_and_sort ("subj_mimetype", where_sql);
                 break;
             case ResultType.LEAST_RECENT_MIMETYPE:
-                sql += group_and_sort ("subj_mimetype", where_sql, true);
+                sql += group_and_sort ("subj_mimetype", where_sql);
                 break;
             case ResultType.MOST_POPULAR_MIMETYPE:
-                sql += group_and_sort ("subj_mimetype", where_sql,
-                    false, false);
+                sql += group_and_sort ("subj_mimetype", where_sql, false);
                 break;
             case ResultType.LEAST_POPULAR_MIMETYPE:
-                sql += group_and_sort ("subj_mimetype", where_sql,
-                    true, true);
+                sql += group_and_sort ("subj_mimetype", where_sql, true);
                 break;
             default:
                 string error_message = "Invalid ResultType.";
@@ -337,6 +298,10 @@ public class DbReader : Object
                 throw new EngineError.INVALID_ARGUMENT (error_message);
         }
 
+        // complete the sort rule
+        bool time_asc = ResultType.is_sort_order_asc ((ResultType) result_type);
+        sql += " timestamp %s".printf ((time_asc) ? "ASC" : "DESC");
+
         int rc;
         Sqlite.Statement stmt;
 
@@ -368,12 +333,27 @@ public class DbReader : Object
             string error_message = "Error in find_event_ids: %d, %s".printf (
                 rc, db.errmsg ());
             warning (error_message);
+            database.assert_not_corrupt (rc);
             throw new EngineError.DATABASE_ERROR (error_message);
         }
 
         return event_ids;
     }
 
+    public uint32[] find_event_ids (TimeRange time_range,
+        GenericArray<Event> event_templates,
+        uint storage_state, uint max_events, uint result_type,
+        BusName? sender=null) throws EngineError
+    {
+        WhereClause where = get_where_clause_for_query (time_range,
+            event_templates, storage_state);
+
+        //if (!where.may_have_results ())
+        //    return new uint32[0];
+
+        return find_event_ids_for_clause (where, max_events, result_type);
+    }
+
     public GenericArray<Event?> find_events (TimeRange time_range,
         GenericArray<Event> event_templates,
         uint storage_state, uint max_events, uint result_type,
@@ -383,6 +363,45 @@ public class DbReader : Object
             storage_state, max_events, result_type));
     }
 
+    public WhereClause get_where_clause_for_query (TimeRange time_range,
+        GenericArray<Event> event_templates, uint storage_state) throws EngineError
+    {
+        WhereClause where = new WhereClause (WhereClause.Type.AND);
+
+        /**
+         * We are using the unary operator here to tell SQLite to not use
+         * the index on the timestamp column at the first place. This is a
+         * "fix" for (LP: #672965) based on some benchmarks, which suggest
+         * a performance win, but we might not oversee all implications.
+         * (See http://www.sqlite.org/optoverview.html, section 6.0).
+         *    -- Markus Korn, 29/11/2010
+         */
+        if (time_range.start != 0)
+            where.add (("+timestamp >= %" + int64.FORMAT).printf(
+                time_range.start));
+        if (time_range.end != 0)
+            where.add (("+timestamp <= %" + int64.FORMAT).printf(
+                time_range.end));
+
+        if (storage_state == StorageState.AVAILABLE ||
+            storage_state == StorageState.NOT_AVAILABLE)
+        {
+            where.add ("(subj_storage_state=? OR subj_storage_state IS NULL)",
+                storage_state.to_string ());
+        }
+        else if (storage_state != StorageState.ANY)
+        {
+            throw new EngineError.INVALID_ARGUMENT(
+                "Unknown storage state '%u'".printf(storage_state));
+        }
+
+        WhereClause tpl_conditions = get_where_clause_from_event_templates (
+            event_templates);
+        where.extend (tpl_conditions);
+
+        return where;
+    }
+
     private struct RelatedUri {
         public uint32 id;
         public int64 timestamp;
@@ -467,14 +486,8 @@ public class DbReader : Object
             // for (int i=0; i<related_uris.length; i++)
             //    related_uris[i] = temp_related_uris[i];
 
-            if (rc != Sqlite.DONE)
-            {
-                string error_message =
-                    "Error in find_related_uris: %d, %s".printf (
-                    rc, db.errmsg ());
-                warning (error_message);
-                throw new EngineError.DATABASE_ERROR (error_message);
-            }
+            database.assert_query_success (rc, "Error in find_related_uris",
+                Sqlite.DONE);
 
             var uri_counter = new HashTable<string, RelatedUri?>(
                 str_hash, str_equal);
@@ -585,10 +598,8 @@ public class DbReader : Object
 
     // Used by find_event_ids
     private string group_and_sort (string field, string where_sql,
-        bool time_asc=false, bool? count_asc=null,
-        string aggregation_type="max")
+        bool? count_asc=null, string aggregation_type="max")
     {
-        string time_sorting = (time_asc) ? "ASC" : "DESC";
         string aggregation_sql = "";
         string order_sql = "";
 
@@ -606,7 +617,7 @@ public class DbReader : Object
                 FROM event_view %s
                 GROUP BY %s)
             GROUP BY %s
-            ORDER BY %s timestamp %s
+            ORDER BY %s 
             """.printf (
                 field,
                 aggregation_type,
@@ -614,11 +625,11 @@ public class DbReader : Object
                 where_sql,
                 field,
                 field,
-                order_sql, time_sorting);
+                order_sql);
     }
 
     // Used by find_event_ids
-    protected WhereClause get_where_clause_from_event_templates (
+    public WhereClause get_where_clause_from_event_templates (
         GenericArray<Event> templates) throws EngineError
     {
         WhereClause where = new WhereClause (WhereClause.Type.OR);
@@ -674,7 +685,7 @@ public class DbReader : Object
                     where.add_wildcard_condition ("actor", val, negated);
                 else
                     where.add_match_condition ("actor",
-                        actors_table.get_id (val), negated);
+                        actors_table.id_try_string (val), negated);
             }
 
             // Origin
@@ -734,7 +745,7 @@ public class DbReader : Object
                             "subj_mimetype", val, negated);
                     else
                         where.add_match_condition ("subj_mimetype",
-                            mimetypes_table.get_id (val), negated);
+                            mimetypes_table.id_try_string (val), negated);
                 }
 
                 // URI
@@ -873,14 +884,14 @@ public class DbReader : Object
         if (symbols.length () == 1)
         {
             subwhere.add_match_condition (table_name,
-                lookup_table.get_id (_symbol));
+                lookup_table.id_try_string (_symbol));
         }
         else
         {
             var sb = new StringBuilder ();
             foreach (unowned string uri in symbols)
             {
-                sb.append_printf ("%d,", lookup_table.get_id (uri));
+                sb.append_printf ("%d,", lookup_table.id_try_string (uri));
             }
             sb.truncate (sb.len - 1);
 
diff --git a/src/engine.vala b/src/engine.vala
index df5c2248..77e711c5 100644
--- a/src/engine.vala
+++ b/src/engine.vala
@@ -212,10 +212,10 @@ public class Engine : DbReader
         insert_stmt.bind_int64 (1, event.id);
         insert_stmt.bind_int64 (2, event.timestamp);
         insert_stmt.bind_int64 (3,
-            interpretations_table.get_id (event.interpretation));
+            interpretations_table.id_for_string (event.interpretation));
         insert_stmt.bind_int64 (4,
-            manifestations_table.get_id (event.manifestation));
-        insert_stmt.bind_int64 (5, actors_table.get_id (event.actor));
+            manifestations_table.id_for_string (event.manifestation));
+        insert_stmt.bind_int64 (5, actors_table.id_for_string (event.actor));
         insert_stmt.bind_text (6, event.origin);
         insert_stmt.bind_int64 (7, payload_id);
 
@@ -228,12 +228,12 @@ public class Engine : DbReader
             insert_stmt.bind_text (8, subject.uri);
             insert_stmt.bind_text (9, subject.current_uri);
             insert_stmt.bind_int64 (10,
-                interpretations_table.get_id (subject.interpretation));
+                interpretations_table.id_for_string (subject.interpretation));
             insert_stmt.bind_int64 (11,
-                manifestations_table.get_id (subject.manifestation));
+                manifestations_table.id_for_string (subject.manifestation));
             insert_stmt.bind_text (12, subject.origin);
             insert_stmt.bind_int64 (13,
-                mimetypes_table.get_id (subject.mimetype));
+                mimetypes_table.id_for_string (subject.mimetype));
             insert_stmt.bind_text (14, subject.text);
             // FIXME: Consider a storages_table table. Too dangerous?
             insert_stmt.bind_text (15, subject.storage);
@@ -241,6 +241,7 @@ public class Engine : DbReader
             if ((rc = insert_stmt.step()) != Sqlite.DONE) {
                 if (rc != Sqlite.CONSTRAINT)
                 {
+                    database.assert_not_corrupt (rc);
                     warning ("SQL error: %d, %s\n", rc, db.errmsg ());
                     return 0;
                 }
@@ -255,12 +256,13 @@ public class Engine : DbReader
 
                 retrieval_stmt.bind_int64 (1, event.timestamp);
                 retrieval_stmt.bind_int64 (2,
-                    interpretations_table.get_id (event.interpretation));
+                    interpretations_table.id_for_string (event.interpretation));
                 retrieval_stmt.bind_int64 (3,
-                    manifestations_table.get_id (event.manifestation));
-                retrieval_stmt.bind_int64 (4, actors_table.get_id (event.actor));
+                    manifestations_table.id_for_string (event.manifestation));
+                retrieval_stmt.bind_int64 (4, actors_table.id_for_string (event.actor));
 
                 if ((rc = retrieval_stmt.step ()) != Sqlite.ROW) {
+                    database.assert_not_corrupt (rc);
                     warning ("SQL error: %d, %s\n", rc, db.errmsg ());
                     return 0;
                 }
@@ -340,6 +342,11 @@ public class Engine : DbReader
             if ((rc = move_stmt.step()) != Sqlite.DONE) {
                 if (rc != Sqlite.CONSTRAINT)
                 {
+                    try
+                    {
+                        database.assert_not_corrupt (rc);
+                    }
+                    catch (EngineError err) {}
                     warning ("SQL error: %d, %s\n", rc, db.errmsg ());
                 }
             }
@@ -364,7 +371,14 @@ public class Engine : DbReader
                 event.payload.data.length);
             if ((rc = payload_insertion_stmt.step ()) != Sqlite.DONE)
                 if (rc != Sqlite.CONSTRAINT)
+                {
                     warning ("SQL error: %d, %s\n", rc, db.errmsg ());
+                    try
+                    {
+                        database.assert_not_corrupt (rc);
+                    }
+                    catch (EngineError err) { }
+                }
 
             return database.database.last_insert_rowid ();
         }
diff --git a/src/errors.vala b/src/errors.vala
index 8c07cfe7..b0c23a7b 100644
--- a/src/errors.vala
+++ b/src/errors.vala
@@ -29,10 +29,11 @@ namespace Zeitgeist
         DATABASE_CORRUPT,
         DATABASE_ERROR,
         DATABASE_RETIRE_FAILED,
+        EXISTING_INSTANCE,
         INVALID_ARGUMENT,
         INVALID_KEY,
-        EXISTING_INSTANCE,
         INVALID_SIGNATURE, // FIXME: change from EngineError to sth. + public
+        TOO_MANY_RESULTS,
     }
 
     // vala doesn't include proper headers, this fixes it
diff --git a/src/extension-store.vala b/src/extension-store.vala
index 82a1a9bb..5ab3775b 100644
--- a/src/extension-store.vala
+++ b/src/extension-store.vala
@@ -81,13 +81,20 @@ namespace Zeitgeist
             storage_stmt.bind_blob (3, data.get_data (), (int) data.get_size ());
 
             if ((rc = storage_stmt.step ()) != Sqlite.DONE)
+            {
+                try
+                {
+                    database.assert_not_corrupt (rc);
+                }
+                catch (EngineError err) { }
                 warning ("SQL error: %d, %s", rc, db.errmsg ());
+            }
         }
 
         /**
          * Retrieve a previously stored value.
          */
-        public Variant? retrieve(string extension, string key, VariantType format)
+        public Variant? retrieve (string extension, string key, VariantType format)
         {
             retrieval_stmt.reset ();
             retrieval_stmt.bind_text (1, extension);
@@ -97,7 +104,14 @@ namespace Zeitgeist
             if (rc != Sqlite.ROW)
             {
                 if (rc != Sqlite.DONE)
+                {
+                    try
+                    {
+                        database.assert_not_corrupt (rc);
+                    }
+                    catch (EngineError err) { }
                     warning ("SQL error: %d, %s", rc, db.errmsg ());
+                }
                 return null;
             }
 
diff --git a/src/mimetype.vala b/src/mimetype.vala
index 55dace93..ae983d78 100644
--- a/src/mimetype.vala
+++ b/src/mimetype.vala
@@ -202,6 +202,7 @@ namespace Zeitgeist
         try {
             register_mimetype ("application/ecmascript", NFO.SOURCE_CODE);
             register_mimetype ("application/javascript", NFO.SOURCE_CODE);
+            register_mimetype ("application/json", NFO.SOURCE_CODE);
             register_mimetype ("application/ms-excel", NFO.SPREADSHEET);
             register_mimetype ("application/ms-powerpoint", NFO.PRESENTATION);
             register_mimetype ("application/msexcel", NFO.SPREADSHEET);
@@ -211,6 +212,7 @@ namespace Zeitgeist
             register_mimetype ("application/postscript", NFO.PAGINATED_TEXT_DOCUMENT);
             register_mimetype ("application/ps", NFO.PAGINATED_TEXT_DOCUMENT);
             register_mimetype ("application/rtf", NFO.PAGINATED_TEXT_DOCUMENT);
+            register_mimetype ("application/soap+xml", NFO.SOURCE_CODE);
             register_mimetype ("application/vnd.corel-draw", NFO.VECTOR_IMAGE);
             register_mimetype ("application/vnd.ms-excel", NFO.SPREADSHEET);
             register_mimetype ("application/vnd.ms-powerpoint", NFO.PRESENTATION);
@@ -236,6 +238,7 @@ namespace Zeitgeist
             register_mimetype ("application/x-gnumeric", NFO.SPREADSHEET);
             register_mimetype ("application/x-gzip", NFO.ARCHIVE);
             register_mimetype ("application/x-java-archive", NFO.SOURCE_CODE);
+            register_mimetype ("application/x-javascript", NFO.SOURCE_CODE);
             register_mimetype ("application/x-killustrator", NFO.VECTOR_IMAGE);
             register_mimetype ("application/x-kpresenter", NFO.PRESENTATION);
             register_mimetype ("application/x-kspread", NFO.SPREADSHEET);
@@ -244,27 +247,37 @@ namespace Zeitgeist
             register_mimetype ("application/x-lzma-compressed-tar", NFO.ARCHIVE);
             register_mimetype ("application/x-m4", NFO.SOURCE_CODE);
             register_mimetype ("application/x-ms-dos-executable", NFO.SOFTWARE);
-            register_mimetype ("application/x-object", NFO.SOURCE_CODE);
             register_mimetype ("application/x-perl", NFO.SOURCE_CODE);
             register_mimetype ("application/x-php", NFO.SOURCE_CODE);
             register_mimetype ("application/x-rpm", NFO.SOFTWARE);
             register_mimetype ("application/x-ruby", NFO.SOURCE_CODE);
             register_mimetype ("application/x-shellscript", NFO.SOURCE_CODE);
+            register_mimetype ("application/x-shockwave-flash", NFO.EXECUTABLE);
             register_mimetype ("application/x-sql", NFO.SOURCE_CODE);
+            register_mimetype ("application/x-stuffit", NFO.ARCHIVE);
             register_mimetype ("application/xhtml+xml", NFO.SOURCE_CODE);
             register_mimetype ("application/xml", NFO.SOURCE_CODE);
+            register_mimetype ("application/xml-dtd", NFO.SOURCE_CODE);
             register_mimetype ("application/zip", NFO.ARCHIVE);
             register_mimetype ("audio/x-scpls", NFO.MEDIA_LIST);
             register_mimetype ("image/gif", NFO.RASTER_IMAGE);
             register_mimetype ("image/jpeg", NFO.RASTER_IMAGE);
+            register_mimetype ("image/pjpeg", NFO.RASTER_IMAGE);
             register_mimetype ("image/png", NFO.RASTER_IMAGE);
             register_mimetype ("image/svg+xml", NFO.VECTOR_IMAGE);
             register_mimetype ("image/tiff", NFO.RASTER_IMAGE);
+            register_mimetype ("image/vnd.microsoft.icon", NFO.ICON);
             register_mimetype ("image/x-xcf", NFO.RASTER_IMAGE);
             register_mimetype ("inode/directory", NFO.FOLDER);
+            register_mimetype ("message/alternative", NMO.EMAIL);
+            register_mimetype ("message/partial", NMO.EMAIL);
+            register_mimetype ("message/related", NMO.EMAIL);
             register_mimetype ("text/css", NFO.SOURCE_CODE);
+            register_mimetype ("text/csv", NFO.TEXT_DOCUMENT);
             register_mimetype ("text/html", NFO.HTML_DOCUMENT);
+            register_mimetype ("text/javascript", NFO.SOURCE_CODE);
             register_mimetype ("text/plain", NFO.TEXT_DOCUMENT);
+            register_mimetype ("text/vcard", NCO.CONTACT);
             register_mimetype ("text/x-c", NFO.SOURCE_CODE);
             register_mimetype ("text/x-c++", NFO.SOURCE_CODE);
             register_mimetype ("text/x-c++src", NFO.SOURCE_CODE);
@@ -280,6 +293,7 @@ namespace Zeitgeist
             register_mimetype ("text/x-haskell", NFO.SOURCE_CODE);
             register_mimetype ("text/x-idl", NFO.SOURCE_CODE);
             register_mimetype ("text/x-java", NFO.SOURCE_CODE);
+            register_mimetype ("text/x-jquery-tmpl", NFO.SOURCE_CODE);
             register_mimetype ("text/x-latex", NFO.SOURCE_CODE);
             register_mimetype ("text/x-lisp", NFO.SOURCE_CODE);
             register_mimetype ("text/x-lua", NFO.SOURCE_CODE);
@@ -296,26 +310,20 @@ namespace Zeitgeist
             register_mimetype ("text/x-troff", NFO.SOURCE_CODE);
             register_mimetype ("text/x-vala", NFO.SOURCE_CODE);
             register_mimetype ("text/x-vhdl", NFO.SOURCE_CODE);
+            register_mimetype ("text/xml", NFO.SOURCE_CODE);
 
             register_mimetype_regex (".*/x-dvi", NFO.PAGINATED_TEXT_DOCUMENT);
-            register_mimetype_regex (
-                "application/vnd.oasis.opendocument.text.*",
-                NFO.PAGINATED_TEXT_DOCUMENT);
-            register_mimetype_regex (
-                "application/vnd.oasis.opendocument.presentation.*",
-                NFO.PRESENTATION);
-            register_mimetype_regex (
-                "application/vnd.oasis.opendocument.spreadsheet.*",
-                NFO.SPREADSHEET);
-            register_mimetype_regex (
-                "application/vnd.oasis.opendocument.graphics.*",
-                NFO.VECTOR_IMAGE);
+            register_mimetype_regex ("application/vnd.ms-excel.*", NFO.SPREADSHEET);
+            register_mimetype_regex ("application/vnd.ms-powerpoint.*", NFO.PRESENTATION);
+            register_mimetype_regex ("application/vnd.oasis.opendocument.graphics.*", NFO.VECTOR_IMAGE);
+            register_mimetype_regex ("application/vnd.oasis.opendocument.presentation.*", NFO.PRESENTATION);
+            register_mimetype_regex ("application/vnd.oasis.opendocument.spreadsheet.*", NFO.SPREADSHEET);
+            register_mimetype_regex ("application/vnd.oasis.opendocument.text.*", NFO.PAGINATED_TEXT_DOCUMENT);
+            register_mimetype_regex ("application/vnd.openxmlformats-officedocument.presentationml.presentation.*", NFO.PRESENTATION);
+            register_mimetype_regex ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.*", NFO.SPREADSHEET);
+            register_mimetype_regex ("application/vnd.openxmlformats-officedocument.wordprocessingml.document.*", NFO.PAGINATED_TEXT_DOCUMENT);
             register_mimetype_regex ("application/vnd\\..*", NFO.DOCUMENT);
             register_mimetype_regex ("application/x-applix-.*", NFO.DOCUMENT);
-            register_mimetype_regex ("application/vnd.ms-excel.*",
-                NFO.SPREADSHEET);
-            register_mimetype_regex ("application/vnd.ms-powerpoint.*",
-                NFO.PRESENTATION);
             register_mimetype_regex ("audio/.*", NFO.AUDIO);
             register_mimetype_regex ("image/.*", NFO.IMAGE);
             register_mimetype_regex ("video/.*", NFO.VIDEO);
diff --git a/src/remote.vala b/src/remote.vala
index 9c780521..af3e5bcf 100644
--- a/src/remote.vala
+++ b/src/remote.vala
@@ -101,7 +101,7 @@ namespace Zeitgeist
         public async abstract void notify_insert (
             [DBus (signature = "(xx)")] Variant time_range,
             [DBus (signature = "a(asaasay)")] Variant events
-        ) throws IOError;
+        ) throws IOError, EngineError;
 
         public async abstract void notify_delete (
             [DBus (signature = "(xx)")] Variant time_range,
diff --git a/src/sql-schema.vala b/src/sql-schema.vala
index 353948ac..4c369a96 100644
--- a/src/sql-schema.vala
+++ b/src/sql-schema.vala
@@ -2,8 +2,9 @@
  *
  * Copyright © 2011 Collabora Ltd.
  *             By Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
- * Copyright © 2011 Canonical Ltd.
+ * Copyright © 2011-2012 Canonical Ltd.
  *             By Michal Hruby <michal.hruby@canonical.com>
+ *             By Siegfried-A. Gevatter <siegfried.gevatter@collabora.co.uk>
  *
  * Based upon a Python implementation (2009-2011) by:
  *  Markus Korn <thekorn@gmx.net>
@@ -37,6 +38,8 @@ namespace Zeitgeist.SQLite
         public const string CORE_SCHEMA = "core";
         public const int CORE_SCHEMA_VERSION = 6;
 
+        private const string DATABASE_CREATION = "database_creation";
+
         public static void ensure_schema (Sqlite.Database database)
             throws EngineError
         {
@@ -47,6 +50,12 @@ namespace Zeitgeist.SQLite
             {
                 // most likely a new DB
                 create_schema (database);
+
+                // set database creation date
+                var schema_sql = ("INSERT INTO schema_version VALUES ('%s', %" +
+                    int64.FORMAT + ")").printf (DATABASE_CREATION,
+                    Timestamp.now ());
+                exec_query (database, schema_sql);
             }
             else if (schema_version == 4 || schema_version == 5)
             {
@@ -120,24 +129,67 @@ namespace Zeitgeist.SQLite
         }
 
         public static int get_schema_version (Sqlite.Database database)
+            throws EngineError
+        {
+            int schema_version = (int) get_schema_metadata (database, CORE_SCHEMA);
+            debug ("schema_version is %d", schema_version);
+
+            if (schema_version < -1)
+            {
+                throw new EngineError.DATABASE_CORRUPT (
+                    "Database corruption flag is set.");
+            }
+            return schema_version;
+        }
+
+        public static int64 get_creation_date (Sqlite.Database database)
+        {
+            return get_schema_metadata (database, DATABASE_CREATION);
+        }
+
+        private static int64 get_schema_metadata (Sqlite.Database database,
+            string key)
+        {
+            var sql = "SELECT version FROM schema_version " +
+                "WHERE schema='%s'".printf (key);
+
+            int64 schema_version = -1;
+
+            database.exec (sql,
+                (n_cols, values, column_names) =>
+                {
+                    if (values[0] != null)
+                    {
+                        schema_version = int64.parse (values[0]);
+                    }
+                    return 0;
+                }, null);
+
+            // we don't really care about the return value of exec, the result
+            // will be -1 if something went wrong anyway
+
+            return schema_version;
+        }
+
+        public static void set_corruption_flag (Sqlite.Database database)
+            throws EngineError
         {
-          var sql = "SELECT version FROM schema_version WHERE schema='core'";
-          int schema_version = -1;
-          database.exec (sql,
-              (n_cols, values, column_names) =>
-              {
-                  if (values[0] != null)
-                  {
-                      schema_version = int.parse (values[0]);
-                  }
-                  return 0;
-              }, null);
-
-          // we don't really care about the return value of exec, the result
-          // will be -1 if something went wrong anyway
-          debug ("schema_version is %d", schema_version);
-
-          return schema_version;
+            // A schema_version value smaller than -1 indicates that
+            // database corruption has been detected.
+            int version = get_schema_version (database);
+            if (version > 0)
+                version = -version;
+            set_schema_version (database, version);
+        }
+
+        private static void set_schema_version (Sqlite.Database database,
+            int schema_version) throws EngineError
+        {
+            /* The 'ON CONFLICT REPLACE' on the PK converts INSERT to UPDATE
+             * when appriopriate */
+            var schema_sql = "INSERT INTO schema_version VALUES ('%s', %d)"
+                .printf (CORE_SCHEMA, schema_version);
+            exec_query (database, schema_sql);
         }
 
         public static void create_schema (Sqlite.Database database)
@@ -458,13 +510,7 @@ namespace Zeitgeist.SQLite
                     version INT
                 )
                 """);
-
-            /* The 'ON CONFLICT REPLACE' on the PK converts INSERT to UPDATE
-             * when appriopriate */
-            var schema_sql = "INSERT INTO schema_version VALUES ('%s', %d)"
-                .printf (CORE_SCHEMA, CORE_SCHEMA_VERSION);
-            exec_query (database, schema_sql);
-
+            set_schema_version (database, CORE_SCHEMA_VERSION);
         }
 
         /**
diff --git a/src/sql.vala b/src/sql.vala
index 09379317..224fde9d 100644
--- a/src/sql.vala
+++ b/src/sql.vala
@@ -4,6 +4,8 @@
  *             By Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
  *             By Seif Lotfy <seif@lotfy.com>
  * Copyright © 2011 Manish Sinha <manishsinha@ubuntu.com>
+ * Copyright © 2012 Canonical Ltd.
+ *             By Siegfried-A. Gevatter <siegfried.gevatter@collabora.co.uk>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -101,12 +103,12 @@ namespace Zeitgeist.SQLite
             int rc = Sqlite.Database.open_v2 (
                 Utils.get_database_file_path (),
                 out database, flags);
-            
+
             if (rc == Sqlite.OK)
             {
                 try
                 {
-                    // Error (like a malformed database) may not be exposed
+                    // Errors (like a malformed database) may not be exposed
                     // until we try to operate on the database.
                     if (is_read_only)
                     {
@@ -134,7 +136,7 @@ namespace Zeitgeist.SQLite
                         throw err;
                 }
             }
-            
+
             if (rc != Sqlite.OK)
             {
                 if (rc == Sqlite.CORRUPT && retry)
@@ -142,17 +144,7 @@ namespace Zeitgeist.SQLite
                     // The database disk image is malformed
                     warning ("It looks like your database is corrupt. " +
                         "It will be renamed and a new one will be created.");
-                    try
-                    {
-                        Utils.retire_database ();
-                    }
-                    catch (Error err)
-                    {
-                        string message =
-                            "Could not rename database: %s".printf (
-                                err.message);
-                        throw new EngineError.DATABASE_RETIRE_FAILED (message);
-                    }
+                    Utils.retire_database ();
                     open_database (false);
                 }
                 else if (rc == Sqlite.PERM || rc == Sqlite.CANTOPEN)
@@ -168,7 +160,7 @@ namespace Zeitgeist.SQLite
                 }
                 else
                 {
-                    string message = "Can't open database: %d, %s".printf(rc,
+                    string message = "Can't open database: %d, %s".printf (rc,
                         database.errmsg ());
                     throw new EngineError.DATABASE_ERROR (message);
                 }
@@ -321,18 +313,49 @@ namespace Zeitgeist.SQLite
          * @param msg message to print if `rc' indicates an error
          * @throws EngineError
          */
+        [Diagnostics]
         public void assert_query_success (int rc, string msg,
             int success_code=Sqlite.OK) throws EngineError
         {
             if (unlikely (rc != success_code))
             {
-                string error_message = "%s: %d, %s".printf(
+                string error_message = "%s: %d, %s".printf (
                     msg, rc, database.errmsg ());
                 warning ("%s\n", error_message);
+                assert_not_corrupt (rc);
                 throw new EngineError.DATABASE_ERROR (error_message);
             }
         }
 
+        /**
+         * Ensure `rc' isn't SQLITE_CORRUPT. If it is, schedule a database
+         * retire and Zeitgeist restart so a new database can be created,
+         * unless in read-only mode, in which case EngineError.DATABASE_ERROR
+         * will be thrown.
+         *
+         * This function should be called whenever assert_query_success isn't
+         * used.
+         *
+         * @param rc error code returned by a SQLite call
+         */
+        public void assert_not_corrupt (int rc)
+            throws EngineError
+        {
+            if (unlikely (rc == Sqlite.CORRUPT))
+            {
+                warning ("It looks like your database is corrupt: %s".printf (
+                    database.errmsg ()));
+                if (!is_read_only)
+                {
+                    // Sets a flag in the database indicating that it is
+                    // corrupt. This will trigger a database retire and
+                    // re-creation on the next startup.
+                    DatabaseSchema.set_corruption_flag (database);
+                }
+                throw new EngineError.DATABASE_CORRUPT (database.errmsg ());
+            }
+        }
+
         private void prepare_read_queries () throws EngineError
         {
             int rc;
diff --git a/src/table-lookup.vala b/src/table-lookup.vala
index c1f09cfa..e56012a0 100644
--- a/src/table-lookup.vala
+++ b/src/table-lookup.vala
@@ -26,7 +26,7 @@ namespace Zeitgeist.SQLite
 
     public class TableLookup : Object
     {
-
+        unowned Zeitgeist.SQLite.Database database;
         unowned Sqlite.Database db;
 
         private string table;
@@ -36,7 +36,9 @@ namespace Zeitgeist.SQLite
         private Sqlite.Statement retrieval_stmt;
 
         public TableLookup (Database database, string table_name)
+            throws EngineError
         {
+            this.database = database;
             db = database.database;
             table = table_name;
             id_to_value = new HashTable<int, string>(direct_hash, direct_equal);
@@ -52,30 +54,38 @@ namespace Zeitgeist.SQLite
                     value_to_id.insert (values[1], int.parse(values[0]));
                     return 0;
                 }, null);
-            if (rc != Sqlite.OK)
-            {
-                critical ("Can't init %s table: %d, %s\n", table,
-                    rc, db.errmsg ());
-            }
+            database.assert_query_success (rc,
+                "Can't init %s table".printf (table));
 
             sql = "INSERT INTO " + table + " (value) VALUES (?)";
             rc = db.prepare_v2 (sql, -1, out insertion_stmt);
-            if (rc != Sqlite.OK)
-            {
-                // FIXME: throw exception and propagate it up to
-                //        zeitgeist-daemon to abort with DB error?
-                critical ("SQL error: %d, %s\n", rc, db.errmsg ());
-            }
+            database.assert_query_success (rc, "Error creating insertion_stmt");
 
             sql = "SELECT value FROM " + table + " WHERE id=?";
             rc = db.prepare_v2 (sql, -1, out retrieval_stmt);
-            if (rc != Sqlite.OK)
-            {
-                critical ("SQL error: %d, %s\n", rc, db.errmsg ());
-            }
+            database.assert_query_success (rc, "Error creating retrieval_stmt");
         }
 
-        public int get_id (string name)
+        /**
+         * Searches the table for the given ID, returns -1 if not found.
+         *
+         * @see id_for_string
+         */
+        public int id_try_string (string name)
+        {
+            int id = value_to_id.lookup (name);
+            if (id == 0)
+                return -1;
+            return id;
+        }
+
+        /**
+         * Searches the table for the given ID, inserts a new one if not found.
+         *
+         * @see id_try_string
+         *
+         */
+        public int id_for_string (string name) throws EngineError
         {
             int id = value_to_id.lookup (name);
             if (id == 0)
@@ -83,10 +93,9 @@ namespace Zeitgeist.SQLite
                 int rc;
                 insertion_stmt.reset ();
                 insertion_stmt.bind_text (1, name);
-                if ((rc = insertion_stmt.step ()) != Sqlite.DONE)
-                {
-                    critical ("SQL error: %d, %s\n", rc, db.errmsg ());
-                }
+                rc = insertion_stmt.step ();
+                database.assert_query_success (rc, "Error in id_for_string",
+                    Sqlite.DONE);
 
                 id = (int) db.last_insert_rowid ();
 
@@ -96,7 +105,7 @@ namespace Zeitgeist.SQLite
             return id;
         }
 
-        public unowned string get_value (int id)
+        public unowned string get_value (int id) throws EngineError
         {
             // When we fetch an event, it either was already in the database
             // at the time Zeitgeist started or it was inserted later -using
@@ -119,7 +128,9 @@ namespace Zeitgeist.SQLite
                 value_to_id.insert (text, id);
                 rc = retrieval_stmt.step ();
             }
-            if (rc != Sqlite.DONE || text == null)
+            database.assert_query_success (rc, "Error in get_value",
+                Sqlite.DONE);
+            if (text == null)
             {
                 critical ("Error getting data from table: %d, %s\n",
                     rc, db.errmsg ());
diff --git a/src/utils.vala b/src/utils.vala
index 033fcfb7..a72b06ea 100644
--- a/src/utils.vala
+++ b/src/utils.vala
@@ -38,6 +38,7 @@ namespace Zeitgeist
         // D-Bus
         public const string DBUS_INTERFACE = "";
         public const string SIG_EVENT = "asaasay";
+        public const size_t MAX_DBUS_RESULT_SIZE = 4 * 1024 * 1024; // 4MiB
 
         // configure runtime cache for events
         // default size is 2000
@@ -129,10 +130,19 @@ namespace Zeitgeist
             original.copy (destination, FileCopyFlags.OVERWRITE, null, null);
         }
 
-        public void retire_database () throws Error
+        public void retire_database () throws EngineError
         {
-            File dbfile = File.new_for_path (get_database_file_path ());
-            dbfile.set_display_name (get_database_file_retire_name ());
+            try
+            {
+                File dbfile = File.new_for_path (get_database_file_path ());
+                dbfile.set_display_name (get_database_file_retire_name ());
+            }
+            catch (Error err)
+            {
+                string message = "Could not rename database: %s".printf (
+                    err.message);
+                throw new EngineError.DATABASE_RETIRE_FAILED (message);
+            }
         }
 
         /**
diff --git a/src/where-clause.vala b/src/where-clause.vala
index fae6bb79..493dd25f 100644
--- a/src/where-clause.vala
+++ b/src/where-clause.vala
@@ -149,7 +149,7 @@ namespace Zeitgeist
             if (!negation)
                 sql = "%s IN (%s)".printf (column, optimized_glob);
             else
-                sql = "%s NOT IN (%s) OR %s is NULL".printf (column,
+                sql = "(%s NOT IN (%s) OR %s is NULL)".printf (column,
                     optimized_glob, column);
             add_with_array (sql, values);
         }
diff --git a/src/zeitgeist-daemon.vala b/src/zeitgeist-daemon.vala
index 1a1effda..c52c1343 100644
--- a/src/zeitgeist-daemon.vala
+++ b/src/zeitgeist-daemon.vala
@@ -138,7 +138,7 @@ namespace Zeitgeist
             var timer = new Timer ();
             GenericArray<Event> events = engine.get_events (event_ids);
             debug ("%s executed in %f seconds", Log.METHOD, timer.elapsed ());
-            return Events.to_variant (events);
+            return Events.to_variant_with_limit (events);
         }
 
         public string[] find_related_uris (Variant time_range,
@@ -176,7 +176,7 @@ namespace Zeitgeist
                 Events.from_variant (event_templates),
                 storage_state, num_events, result_type, sender);
             debug ("%s executed in %f seconds", Log.METHOD, timer.elapsed ());
-            return Events.to_variant (events);
+            return Events.to_variant_with_limit (events);
         }
 
         public uint32[] insert_events (
diff --git a/test/dbus/engine-test.py b/test/dbus/engine-test.py
index 91d55a03..7217a9ba 100644
--- a/test/dbus/engine-test.py
+++ b/test/dbus/engine-test.py
@@ -134,7 +134,7 @@ class ZeitgeistEngineTest(testutils.RemoteTestCase):
 
 	def testIllegalPredefinedEventId(self):
 		event = Event()
-		event[0][0] = str(23) # This is illegal, we assert the erro later
+		event[0][0] = str(23) # This is illegal, we assert the error later
 		event.timestamp = 0
 		event.interpretation = Manifestation.USER_ACTIVITY
 		event.manifestation = Interpretation.CREATE_EVENT
diff --git a/test/direct/table-lookup-test.vala b/test/direct/table-lookup-test.vala
index 759fbe1b..457aa42d 100644
--- a/test/direct/table-lookup-test.vala
+++ b/test/direct/table-lookup-test.vala
@@ -57,16 +57,16 @@ public void basic_test ()
     unowned Sqlite.Database db = database.database;
     TableLookup table_lookup = new TableLookup (database, "actor");
 
-    int id = table_lookup.get_id ("1st-actor");
-    assert_cmpint (table_lookup.get_id ("2nd-actor"), OperatorType.EQUAL, id+1);
-    assert_cmpint (table_lookup.get_id ("1st-actor"), OperatorType.EQUAL, id);
+    int id = table_lookup.id_for_string ("1st-actor");
+    assert_cmpint (table_lookup.id_for_string ("2nd-actor"), OperatorType.EQUAL, id+1);
+    assert_cmpint (table_lookup.id_for_string ("1st-actor"), OperatorType.EQUAL, id);
 
     int rc = db.exec ("DELETE FROM actor WHERE value='1st-actor'");
     assert (rc == Sqlite.OK);
 
     table_lookup.remove (1);
-    assert_cmpint (table_lookup.get_id ("2nd-actor"), OperatorType.EQUAL, id+1);
-    assert_cmpint (table_lookup.get_id ("1st-actor"), OperatorType.EQUAL, id+2);
+    assert_cmpint (table_lookup.id_for_string ("2nd-actor"), OperatorType.EQUAL, id+1);
+    assert_cmpint (table_lookup.id_for_string ("1st-actor"), OperatorType.EQUAL, id+2);
 }
 
 public void get_value_with_query_test ()
@@ -88,14 +88,14 @@ public void engine_test ()
     unowned Sqlite.Database db = database.database;
     TableLookup table_lookup = engine.get_actors_table_lookup();
 
-    assert_cmpint (table_lookup.get_id ("something"), OperatorType.EQUAL, 1);
+    assert_cmpint (table_lookup.id_for_string ("something"), OperatorType.EQUAL, 1);
 
     // Since we're running with Engine, this should trigger the deletion
     // callback, which in turn should fix the cache (LP: #598666).
     int rc = db.exec ("DELETE FROM actor WHERE value='something'");
     assert (rc == Sqlite.OK);
 
-    assert_cmpint (table_lookup.get_id ("something"), OperatorType.EQUAL, 2);
+    assert_cmpint (table_lookup.id_for_string ("something"), OperatorType.EQUAL, 2);
 }
 
 // vim:expandtab:ts=4:sw=4