Load slurp config from toml file

.cargo/config.toml

@@ -2,8 +2,3 @@
 [build]
 rustflags = [ "--cfg=web_sys_unstable_apis" ]
 
-[registry]
-global-credential-providers = ["cargo:token"]
-
-[registries.xinu]
-index = "sparse+https://git.z.xinu.tv/api/packages/wathiede/cargo/"

Cargo.toml

@@ -1,10 +1,15 @@
 [workspace]
 resolver = "2"
-default-members = ["server"]
+default-members = [
-members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+  "server"
-
+]
-[profile.dev]
+members = [
-opt-level = 1
+  "web",
+  "server",
+  "notmuch",
+  "procmail2notmuch",
+  "shared"
+]
 
 [profile.release]
 lto = true

Makefile

@@ -1,6 +1,5 @@
 .PHONEY: release
 release:
-	(cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true)
 	bash scripts/update-crate-version.sh
 	git push
 

notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "notmuch"
-version = "0.0.115"
+version = "0.0.29"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -10,7 +10,6 @@ log = "0.4.14"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
 thiserror = "1.0.30"
-tracing = "0.1.41"
 
 [dev-dependencies]
 itertools = "0.10.1"

notmuch/src/lib.rs

@@ -213,9 +213,8 @@ use std::{
     process::Command,
 };
 
-use log::{error, info};
+use log::info;
 use serde::{Deserialize, Serialize};
-use tracing::instrument;
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -466,8 +465,6 @@ pub struct Notmuch {
     config_path: Option<PathBuf>,
 }
 
-// TODO: rewrite to use tokio::process::Command and make everything async to see if that helps with
-// concurrency being more parallel.
 impl Notmuch {
     pub fn with_config<P: AsRef<Path>>(config_path: P) -> Notmuch {
         Notmuch {
@@ -475,7 +472,6 @@ impl Notmuch {
         }
     }
 
-    #[instrument(skip_all)]
     pub fn new(&self) -> Result<Vec<u8>, NotmuchError> {
         self.run_notmuch(["new"])
     }
@@ -484,7 +480,6 @@ impl Notmuch {
         self.run_notmuch(std::iter::empty::<&str>())
     }
 
-    #[instrument(skip_all, fields(query=query))]
     pub fn tags_for_query(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
         let res = self.run_notmuch(["search", "--format=json", "--output=tags", query])?;
         Ok(serde_json::from_slice(&res)?)
@@ -494,19 +489,16 @@ impl Notmuch {
         self.tags_for_query("*")
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_add(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
         self.run_notmuch(["tag", &format!("+{tag}"), search_term])?;
         Ok(())
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_remove(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
         self.run_notmuch(["tag", &format!("-{tag}"), search_term])?;
         Ok(())
     }
 
-    #[instrument(skip_all, fields(query=query,offset=offset,limit=limit))]
     pub fn search(
         &self,
         query: &str,
@@ -515,35 +507,24 @@ impl Notmuch {
     ) -> Result<SearchSummary, NotmuchError> {
         let query = if query.is_empty() { "*" } else { query };
 
-        let res = self
+        let res = self.run_notmuch([
-            .run_notmuch([
+            "search",
-                "search",
+            "--format=json",
-                "--format=json",
+            &format!("--offset={offset}"),
-                &format!("--offset={offset}"),
+            &format!("--limit={limit}"),
-                &format!("--limit={limit}"),
+            query,
-                query,
+        ])?;
-            ])
+        Ok(serde_json::from_slice(&res)?)
-            .inspect_err(|err| error!("failed to notmuch search for query '{query}': {err}"))?;
-        Ok(serde_json::from_slice(&res).unwrap_or_else(|err| {
-            error!("failed to decode search result for query '{query}': {err}");
-            SearchSummary(Vec::new())
-        }))
     }
 
-    #[instrument(skip_all, fields(query=query))]
     pub fn count(&self, query: &str) -> Result<usize, NotmuchError> {
-        // NOTE: --output=threads is technically more correct, but really slow
+        // TODO: compare speed of notmuch count for * w/ and w/o --output=threads
-        // TODO: find a fast thread count path
+        let res = self.run_notmuch(["count", "--output=threads", query])?;
-        // let res = self.run_notmuch(["count", "--output=threads", query])?;
-        let res = self.run_notmuch(["count", query])?;
         // Strip '\n' from res.
-        let s = std::str::from_utf8(&res)?.trim();
+        let s = std::str::from_utf8(&res[..res.len() - 1])?;
-        Ok(s.parse()
+        Ok(s.parse()?)
-            .inspect_err(|err| error!("failed to parse count for query '{query}': {err}"))
-            .unwrap_or(0))
     }
 
-    #[instrument(skip_all, fields(query=query))]
     pub fn show(&self, query: &str) -> Result<ThreadSet, NotmuchError> {
         let slice = self.run_notmuch([
             "show",
@@ -562,7 +543,6 @@ impl Notmuch {
         Ok(val)
     }
 
-    #[instrument(skip_all, fields(query=query,part=part))]
     pub fn show_part(&self, query: &str, part: usize) -> Result<Part, NotmuchError> {
         let slice = self.run_notmuch([
             "show",
@@ -582,24 +562,20 @@ impl Notmuch {
         Ok(val)
     }
 
-    #[instrument(skip_all, fields(id=id))]
     pub fn show_original(&self, id: &MessageId) -> Result<Vec<u8>, NotmuchError> {
         self.show_original_part(id, 0)
     }
 
-    #[instrument(skip_all, fields(id=id,part=part))]
     pub fn show_original_part(&self, id: &MessageId, part: usize) -> Result<Vec<u8>, NotmuchError> {
         let res = self.run_notmuch(["show", "--part", &part.to_string(), id])?;
         Ok(res)
     }
 
-    #[instrument(skip_all, fields(query=query))]
     pub fn message_ids(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
         let res = self.run_notmuch(["search", "--output=messages", "--format=json", query])?;
         Ok(serde_json::from_slice(&res)?)
     }
 
-    #[instrument(skip_all, fields(query=query))]
     pub fn files(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
         let res = self.run_notmuch(["search", "--output=files", "--format=json", query])?;
         Ok(serde_json::from_slice(&res)?)

procmail2notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "procmail2notmuch"
-version = "0.0.115"
+version = "0.0.29"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

server/.sqlx/query-113694cd5bf0d2582ff3a635776daa608fe88abe1185958c4215646c92335afb.json

@@ -52,7 +52,7 @@
     "nullable": [
       true,
       true,
-      false,
+      true,
       true,
       true,
       true,

server/.sqlx/query-126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nSELECT\n  url\nFROM email_photo ep\nJOIN email_address ea\nON ep.id = ea.email_photo_id\nWHERE\n  address = $1\n    ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "url",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef"
-}

server/.sqlx/query-1b2244c9b9b64a1395d8d266f5df5352242bbe5efe481b0852e1c1d4b40584a7.json

@@ -0,0 +1,62 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    site,\n    title,\n    summary,\n    link,\n    date,\n    is_read,\n    uid,\n    id\nFROM post\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "link",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "date",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_read",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "uid",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "1b2244c9b9b64a1395d8d266f5df5352242bbe5efe481b0852e1c1d4b40584a7"
+}

server/.sqlx/query-2c1954b6db3cbcabf9b878cd1c8ea01c607f46dc43a85b58e19217e7633cf337.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $5 :: text IS NULL\n      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $5)\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
+  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
   "describe": {
     "columns": [
       {
@@ -39,8 +39,7 @@
         "Text",
         "Bool",
         "Int8",
-        "Int8",
+        "Int8"
-        "Text"
       ]
     },
     "nullable": [
@@ -52,5 +51,5 @@
       true
     ]
   },
-  "hash": "99114d4840067acb12d9a41ef036bdd8ecf87cfdde8ce4985821485816af5213"
+  "hash": "2c1954b6db3cbcabf9b878cd1c8ea01c607f46dc43a85b58e19217e7633cf337"
 }

server/.sqlx/query-dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nSELECT id\nFROM feed\nWHERE slug = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc"
-}

server/.sqlx/query-e118f546c628661023aa25803bb29affb6cd25eca63246e5ace5b90a845d76ac.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n    COUNT(*) count\nFROM\n    post\nWHERE\n    (\n        $1 :: text IS NULL\n        OR site = $1\n    )\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $3 :: text IS NULL\n      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $3)\n    )\n",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Bool",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "e118f546c628661023aa25803bb29affb6cd25eca63246e5ace5b90a845d76ac"
-}

server/.sqlx/query-e28b890e308f483aa6bd08617548ae66294ae1e99b1cab49f5f4211e0fd7d419.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    COUNT(*) count\nFROM\n    post\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e28b890e308f483aa6bd08617548ae66294ae1e99b1cab49f5f4211e0fd7d419"
+}

server/.sqlx/query-e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nINSERT INTO feed ( name, slug, url, homepage, selector )\nVALUES ( $1, $2, $3, '', '' )\nRETURNING id\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd"
-}

server/Cargo.toml

@@ -1,22 +1,21 @@
 [package]
-name = "letterbox-server"
+name = "server"
-version = "0.0.115"
+version = "0.0.29"
 edition = "2021"
-default-run = "letterbox-server"
+default-run = "server"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 ammonia = "3.3.0"
 anyhow = "1.0.79"
-async-graphql = { version = "7", features = ["log"] }
+async-graphql = { version = "6.0.11", features = ["log"] }
-async-graphql-rocket = "7"
+async-graphql-rocket = "6.0.11"
 async-trait = "0.1.81"
 build-info = "0.0.38"
-cacher = { version = "0.1.0", registry = "xinu" }
+cacher = {git = "http://git-private.h.xinu.tv/wathiede/cacher.git"}
-chrono = "0.4.39"
-clap = { version = "4.5.23", features = ["derive"] }
 css-inline = "0.13.0"
+glog = "0.1.0"
 html-escape = "0.2.13"
 linkify = "0.10.0"
 log = "0.4.17"
@@ -25,28 +24,19 @@ mailparse = "0.15.0"
 maplit = "1.0.2"
 memmap = "0.7.0"
 notmuch = { path = "../notmuch" }
-opentelemetry = "0.27.1"
 reqwest = { version = "0.12.7", features = ["blocking"] }
-rocket = { version = "0.5.0-rc.2", features = ["json"] }
+rocket = { version = "0.5.0-rc.2", features = [ "json" ] }
 rocket_cors = "0.6.0"
 scraper = "0.20.0"
 serde = { version = "1.0.147", features = ["derive"] }
 serde_json = "1.0.87"
 shared = { path = "../shared" }
-sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
+sqlx = { version = "0.7.4", features = ["postgres", "runtime-tokio", "time"] }
-tantivy = { version = "0.22.0", optional = true }
+tantivy = "0.22.0"
 thiserror = "1.0.37"
 tokio = "1.26.0"
-tracing = "0.1.41"
 url = "2.5.2"
 urlencoding = "2.1.3"
-#xtracing = { path = "../../xtracing" }
-#xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
-xtracing = { version = "0.1.0", registry = "xinu" }
 
 [build-dependencies]
 build-info-build = "0.0.38"
-
-[features]
-#default = [ "tantivy" ]
-tantivy = ["dep:tantivy"]

server/Rocket.toml

@@ -12,3 +12,43 @@ port = 9345
 newsreader_database_url = "postgres://newsreader@nixos-07.h.xinu.tv/newsreader"
 newsreader_tantivy_db_path = "../target/database/newsreader"
 slurp_cache_path = "/net/nasx/x/letterbox/slurp"
+
+[debug.slurp_site_selectors]
+"atmeta.com" = [
+  "div.entry-content"
+]
+"blog.prusa3d.com" = [
+  "article.content .post-block"
+]
+"blog.cloudflare.com" = [
+  ".author-lists .author-name-tooltip",
+  ".post-full-content"
+]
+"blog.zsa.io" = [
+  "section.blog-article"
+]
+"engineering.fb.com" = [
+  "article"
+]
+"hackaday.com" = [
+  "div.entry-featured-image",
+  "div.entry-content"
+]
+"mitchellh.com" = [
+  "div.w-full"
+]
+"natwelch.com" = [
+  "article div.prose"
+]
+"slashdot.org" = [
+  "span.story-byline",
+  "div.p"
+]
+"www.redox-os.org" = [
+  "div.content"
+]
+"www.smbc-comics.com" = [
+  "img#cc-comic",
+  "div#aftercomic img"
+]
+

server/build.rs

@@ -1,5 +1,5 @@
 fn main() {
-    // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
+	// Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
-    // and `build_info::format!` in the main program.
+	// and `build_info::format!` in the main program.
-    build_info_build::build_script();
+	build_info_build::build_script();
 }

server/migrations/20241217021645_create-search-table.down.sql

@@ -1,3 +0,0 @@
-DROP INDEX IF EXISTS post_summary_idx;
-DROP INDEX IF EXISTS post_site_idx;
-DROP INDEX IF EXISTS post_title_idx;

server/migrations/20241217021645_create-search-table.up.sql

@@ -1,3 +0,0 @@
-CREATE INDEX post_summary_idx ON post USING GIN (to_tsvector('english', summary));
-CREATE INDEX post_site_idx ON post USING GIN (to_tsvector('english', site));
-CREATE INDEX post_title_idx ON post USING GIN (to_tsvector('english', title));

server/migrations/20241218010438_create-email-tables.down.sql

@@ -1,24 +0,0 @@
-BEGIN;
-
-ALTER TABLE IF EXISTS public."Email" DROP CONSTRAINT IF EXISTS email_avatar_fkey;
-ALTER TABLE IF EXISTS public."EmailDisplayName" DROP CONSTRAINT IF EXISTS email_id_fk;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_to_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_cc_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_from_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_header_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_file_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_body_id_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_thread_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_tag_fkey;
-
-DROP TABLE IF EXISTS public."Email";
-DROP TABLE IF EXISTS public."EmailDisplayName";
-DROP TABLE IF EXISTS public."Message";
-DROP TABLE IF EXISTS public."Header";
-DROP TABLE IF EXISTS public."File";
-DROP TABLE IF EXISTS public."Avatar";
-DROP TABLE IF EXISTS public."Body";
-DROP TABLE IF EXISTS public."Thread";
-DROP TABLE IF EXISTS public."Tag";
-
-END;

server/migrations/20241218010438_create-email-tables.up.sql

@@ -1,174 +0,0 @@
--- This script was generated by the ERD tool in pgAdmin 4.
--- Please log an issue at https://github.com/pgadmin-org/pgadmin4/issues/new/choose if you find any bugs, including reproduction steps.
-BEGIN;
-
-ALTER TABLE IF EXISTS public."Email" DROP CONSTRAINT IF EXISTS email_avatar_fkey;
-ALTER TABLE IF EXISTS public."EmailDisplayName" DROP CONSTRAINT IF EXISTS email_id_fk;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_to_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_cc_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_from_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_header_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_file_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_body_id_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_thread_fkey;
-ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_tag_fkey;
-
-CREATE TABLE IF NOT EXISTS public."Email"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    address text NOT NULL,
-    avatar_id integer,
-    PRIMARY KEY (id),
-    CONSTRAINT avatar_id UNIQUE (avatar_id)
-);
-
-CREATE TABLE IF NOT EXISTS public."EmailDisplayName"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    email_id integer NOT NULL,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Message"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    subject text,
-    "from" integer,
-    "to" integer,
-    cc integer,
-    header_id integer,
-    hash text NOT NULL,
-    file_id integer NOT NULL,
-    date timestamp with time zone NOT NULL,
-    unread boolean NOT NULL,
-    body_id integer NOT NULL,
-    thread_id integer NOT NULL,
-    tag_id integer,
-    CONSTRAINT body_id UNIQUE (body_id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Header"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    key text NOT NULL,
-    value text NOT NULL,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."File"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    path text NOT NULL,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Avatar"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    url text NOT NULL,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Body"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    text text NOT NULL,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Thread"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS public."Tag"
-(
-    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
-    name text NOT NULL,
-    display text,
-    fg_color integer,
-    bg_color integer,
-    PRIMARY KEY (id)
-);
-
-ALTER TABLE IF EXISTS public."Email"
-    ADD CONSTRAINT email_avatar_fkey FOREIGN KEY (avatar_id)
-    REFERENCES public."Avatar" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."EmailDisplayName"
-    ADD CONSTRAINT email_id_fk FOREIGN KEY (email_id)
-    REFERENCES public."Email" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_to_fkey FOREIGN KEY ("to")
-    REFERENCES public."Email" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_cc_fkey FOREIGN KEY (cc)
-    REFERENCES public."Email" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_from_fkey FOREIGN KEY ("from")
-    REFERENCES public."Email" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_header_fkey FOREIGN KEY (header_id)
-    REFERENCES public."Header" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_file_fkey FOREIGN KEY (file_id)
-    REFERENCES public."File" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_body_id_fkey FOREIGN KEY (body_id)
-    REFERENCES public."Body" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_thread_fkey FOREIGN KEY (thread_id)
-    REFERENCES public."Thread" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-
-ALTER TABLE IF EXISTS public."Message"
-    ADD CONSTRAINT message_tag_fkey FOREIGN KEY (tag_id)
-    REFERENCES public."Tag" (id) MATCH SIMPLE
-    ON UPDATE NO ACTION
-    ON DELETE NO ACTION
-    NOT VALID;
-
-END;

server/migrations/20241225054216_post-link-unique.down.sql

@@ -1,3 +0,0 @@
--- Add down migration script here
-ALTER TABLE
-    post DROP CONSTRAINT post_link_key;

server/migrations/20241225054216_post-link-unique.up.sql

@@ -1,28 +0,0 @@
-WITH dupes AS (
-    SELECT
-        uid,
-        link,
-        Row_number() over(
-            PARTITION by link
-            ORDER BY
-                link
-        ) AS RowNumber
-    FROM
-        post
-)
-DELETE FROM
-    post
-WHERE
-    uid IN (
-        SELECT
-            uid
-        FROM
-            dupes
-        WHERE
-            RowNumber > 1
-    );
-
-ALTER TABLE
-    post
-ADD
-    UNIQUE (link);

server/migrations/20241225180250_post-link-non-empty.down.sql

@@ -1,7 +0,0 @@
-ALTER TABLE
-    post
-ALTER COLUMN
-    link DROP NOT NULL;
-
-ALTER TABLE
-    post DROP CONSTRAINT link;

server/migrations/20241225180250_post-link-non-empty.up.sql

@@ -1,17 +0,0 @@
-DELETE FROM
-    post
-WHERE
-    link IS NULL
-    OR link = '';
-
-ALTER TABLE
-    post
-ALTER COLUMN
-    link
-SET
-    NOT NULL;
-
-ALTER TABLE
-    post
-ADD
-    CONSTRAINT link CHECK (link <> '');

server/migrations/20250114045217_create-email-photos.down.sql

@@ -1,3 +0,0 @@
-DROP TABLE IF EXISTS email_address;
-DROP TABLE IF EXISTS photo;
-DROP TABLE IF EXISTS google_person;

server/migrations/20250114045217_create-email-photos.up.sql

@@ -1,19 +0,0 @@
--- Add up migration script here
-CREATE TABLE IF NOT EXISTS google_person (
-  id SERIAL PRIMARY KEY,
-  resource_name TEXT NOT NULL UNIQUE,
-  display_name TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS email_photo (
-  id SERIAL PRIMARY KEY,
-  google_person_id INTEGER REFERENCES google_person (id) UNIQUE,
-  url TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS email_address (
-  id SERIAL PRIMARY KEY,
-  address TEXT NOT NULL UNIQUE,
-  email_photo_id INTEGER REFERENCES email_photo (id),
-  google_person_id INTEGER REFERENCES google_person (id)
-);

server/migrations/20250117225159_strip-html-index.down.sql

@@ -1,5 +0,0 @@
--- Add down migration script here
-DROP INDEX post_summary_idx;
-CREATE INDEX post_summary_idx ON post USING gin (
-    to_tsvector('english', summary)
-);

server/migrations/20250117225159_strip-html-index.up.sql

@@ -1,11 +0,0 @@
--- Something like this around summary in the idx w/ tsvector
-DROP INDEX post_summary_idx;
-CREATE INDEX post_summary_idx ON post USING gin (to_tsvector(
-    'english',
-    regexp_replace(
-        regexp_replace(summary, '<[^>]+>', ' ', 'g'),
-        '\s+',
-        ' ',
-        'g'
-    )
-));

server/sql/all-posts.sql

@@ -6,9 +6,5 @@ SELECT
     date,
     is_read,
     uid,
-    p.id id
+    id
-FROM
+FROM post
-    post AS p
-    JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
-ORDER BY
-    date DESC;

server/sql/all-uids.sql

@@ -1,6 +0,0 @@
-SELECT
-    uid
-FROM
-    post AS p
-    JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
-;

server/sql/count.sql

@@ -3,15 +3,8 @@ SELECT
 FROM
     post
 WHERE
-    (
+    ($1::text IS NULL OR site = $1)
-        $1 :: text IS NULL
-        OR site = $1
-    )
     AND (
         NOT $2
         OR NOT is_read
     )
-    AND (
-      $3 :: text IS NULL
-      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $3)
-    )

server/sql/posts-from-uids.sql

@@ -1,14 +0,0 @@
-SELECT
-    site AS "site!",
-    title AS "title!",
-    summary AS "summary!",
-    link AS "link!",
-    date AS "date!",
-    is_read AS "is_read!",
-    uid AS "uid!",
-    p.id id
-FROM
-    post p
-    JOIN feed f ON p.site = f.slug
-WHERE
-    uid = ANY ($1);

server/sql/threads-from-uid.sql

@@ -1,14 +0,0 @@
-SELECT
-    site,
-    date,
-    is_read,
-    title,
-    uid,
-    name
-FROM
-    post p
-    JOIN feed f ON p.site = f.slug
-WHERE
-    uid = ANY ($1)
-ORDER BY
-    date DESC;

server/sql/threads.sql

@@ -14,10 +14,6 @@ WHERE
         NOT $2
         OR NOT is_read
     )
-    AND (
-      $5 :: text IS NULL
-      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $5)
-    )
 ORDER BY
     date DESC,
     title OFFSET $3

server/src/bin/cleanhtml.rs

@@ -1,6 +1,6 @@
 use std::fs;
 
-use letterbox_server::sanitize_html;
+use server::sanitize_html;
 
 fn main() -> anyhow::Result<()> {
     let mut args = std::env::args().skip(1);

server/src/bin/email2db.rs

@@ -1,22 +0,0 @@
-use clap::Parser;
-use letterbox_server::mail::read_mail_to_db;
-use sqlx::postgres::PgPool;
-
-/// Add certain emails as posts in newsfeed app.
-#[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-    /// DB URL, something like postgres://newsreader@nixos-07.h.xinu.tv/newsreader
-    #[arg(short, long)]
-    db_url: String,
-    /// path to parse
-    path: String,
-}
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
-    let args = Args::parse();
-    let pool = PgPool::connect(&args.db_url).await?;
-    read_mail_to_db(&pool, &args.path).await?;
-    Ok(())
-}

server/src/bin/server.rs

@@ -5,16 +5,9 @@
 extern crate rocket;
 use std::{error::Error, io::Cursor, str::FromStr};
 
-use async_graphql::{extensions, http::GraphiQLSource, EmptySubscription, Schema};
+use async_graphql::{http::GraphiQLSource, EmptySubscription, Schema};
 use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
-#[cfg(feature = "tantivy")]
+use glog::Flags;
-use letterbox_server::tantivy::TantivyConnection;
-use letterbox_server::{
-    config::Config,
-    error::ServerError,
-    graphql::{Attachment, GraphqlSchema, Mutation, QueryRoot},
-    nm::{attachment_bytes, cid_attachment_bytes},
-};
 use notmuch::{Notmuch, NotmuchError, ThreadSet};
 use rocket::{
     fairing::AdHoc,
@@ -25,7 +18,19 @@ use rocket::{
     Response, State,
 };
 use rocket_cors::{AllowedHeaders, AllowedOrigins};
+use server::{
+    config::Config,
+    error::ServerError,
+    graphql::{Attachment, GraphqlSchema, Mutation, QueryRoot},
+    nm::{attachment_bytes, cid_attachment_bytes},
+};
 use sqlx::postgres::PgPool;
+use tantivy::{Index, IndexWriter};
+
+#[get("/refresh")]
+async fn refresh(nm: &State<Notmuch>) -> Result<Json<String>, Debug<NotmuchError>> {
+    Ok(Json(String::from_utf8_lossy(&nm.new()?).to_string()))
+}
 
 #[get("/show/<query>/pretty")]
 async fn show_pretty(
@@ -161,6 +166,126 @@ fn graphiql() -> content::RawHtml<String> {
     content::RawHtml(GraphiQLSource::build().endpoint("/api/graphql").finish())
 }
 
+#[rocket::post("/create-news-db")]
+fn create_news_db(config: &State<Config>) -> Result<String, Debug<ServerError>> {
+    create_news_db_impl(config)?;
+    Ok(format!(
+        "DB created in {}\n",
+        config.newsreader_tantivy_db_path
+    ))
+}
+fn create_news_db_impl(config: &Config) -> Result<(), ServerError> {
+    std::fs::remove_dir_all(&config.newsreader_tantivy_db_path).map_err(ServerError::from)?;
+    std::fs::create_dir_all(&config.newsreader_tantivy_db_path).map_err(ServerError::from)?;
+    use tantivy::schema::*;
+    let mut schema_builder = Schema::builder();
+    schema_builder.add_text_field("site", STRING | STORED);
+    schema_builder.add_text_field("title", TEXT | STORED);
+    schema_builder.add_text_field("summary", TEXT);
+    schema_builder.add_text_field("link", STRING | STORED);
+    schema_builder.add_date_field("date", FAST);
+    schema_builder.add_bool_field("is_read", FAST);
+    schema_builder.add_text_field("uid", STRING | STORED);
+    schema_builder.add_i64_field("id", FAST);
+
+    let schema = schema_builder.build();
+    Index::create_in_dir(&config.newsreader_tantivy_db_path, schema).map_err(ServerError::from)?;
+    Ok(())
+}
+
+#[rocket::post("/reindex-news-db")]
+async fn reindex_news_db(
+    pool: &State<PgPool>,
+    config: &State<Config>,
+) -> Result<String, Debug<ServerError>> {
+    use tantivy::{doc, Term};
+
+    let start_time = std::time::Instant::now();
+    let pool: &PgPool = pool;
+
+    let index =
+        Index::open_in_dir(&config.newsreader_tantivy_db_path).map_err(ServerError::from)?;
+    let mut index_writer = index.writer(50_000_000).map_err(ServerError::from)?;
+    let schema = index.schema();
+    let site = schema.get_field("site").map_err(ServerError::from)?;
+    let title = schema.get_field("title").map_err(ServerError::from)?;
+    let summary = schema.get_field("summary").map_err(ServerError::from)?;
+    let link = schema.get_field("link").map_err(ServerError::from)?;
+    let date = schema.get_field("date").map_err(ServerError::from)?;
+    let is_read = schema.get_field("is_read").map_err(ServerError::from)?;
+    let uid = schema.get_field("uid").map_err(ServerError::from)?;
+    let id = schema.get_field("id").map_err(ServerError::from)?;
+
+    let rows = sqlx::query_file!("sql/all-posts.sql")
+        .fetch_all(pool)
+        .await
+        .map_err(ServerError::from)?;
+
+    let total = rows.len();
+    for (i, r) in rows.into_iter().enumerate() {
+        if i % 10_000 == 0 {
+            info!(
+                "{i}/{total} processed, elapsed {:.2}s",
+                start_time.elapsed().as_secs_f32()
+            );
+        }
+        let id_term = Term::from_field_text(uid, &r.uid);
+        index_writer.delete_term(id_term);
+        index_writer
+            .add_document(doc!(
+                site =>  r.site.expect("UNKOWN_SITE"),
+                title =>  r.title.expect("UNKOWN_TITLE"),
+                // TODO: clean and extract text from HTML
+                summary => r.summary.expect("UNKNOWN_SUMMARY"),
+                link => r.link.expect("link"),
+                date => tantivy::DateTime::from_primitive(r.date.expect("date")),
+                is_read => r.is_read.expect("is_read"),
+                uid => r.uid,
+                id => r.id as i64,
+            ))
+            .map_err(ServerError::from)?;
+    }
+
+    index_writer.commit().map_err(ServerError::from)?;
+
+    info!("took {:.2}s to reindex", start_time.elapsed().as_secs_f32());
+    Ok(format!(
+        "DB openned in {}\n",
+        config.newsreader_tantivy_db_path
+    ))
+}
+
+#[rocket::get("/search-news-db")]
+fn search_news_db(
+    index: &State<tantivy::Index>,
+    reader: &State<tantivy::IndexReader>,
+) -> Result<String, Debug<ServerError>> {
+    use tantivy::{collector::TopDocs, query::QueryParser, Document, TantivyDocument};
+
+    let searcher = reader.searcher();
+    let schema = index.schema();
+    let site = schema.get_field("site").map_err(ServerError::from)?;
+    let title = schema.get_field("title").map_err(ServerError::from)?;
+    let summary = schema.get_field("summary").map_err(ServerError::from)?;
+    let query_parser = QueryParser::for_index(&index, vec![site, title, summary]);
+
+    let query = query_parser
+        .parse_query("grapheme")
+        .map_err(ServerError::from)?;
+    let top_docs = searcher
+        .search(&query, &TopDocs::with_limit(10))
+        .map_err(ServerError::from)?;
+    let mut results = vec![];
+    info!("search found {} docs", top_docs.len());
+    for (_score, doc_address) in top_docs {
+        let retrieved_doc: TantivyDocument =
+            searcher.doc(doc_address).map_err(ServerError::from)?;
+        results.push(format!("{}", retrieved_doc.to_json(&schema)));
+    }
+
+    Ok(format!("{}", results.join(" ")))
+}
+
 #[rocket::get("/graphql?<query..>")]
 async fn graphql_query(schema: &State<GraphqlSchema>, query: GraphQLQuery) -> GraphQLResponse {
     query.execute(schema.inner()).await
@@ -176,7 +301,14 @@ async fn graphql_request(
 
 #[rocket::main]
 async fn main() -> Result<(), Box<dyn Error>> {
-    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
+    glog::new()
+        .init(Flags {
+            colorlogtostderr: true,
+            //alsologtostderr: true, // use logtostderr to only write to stderr and not to files
+            logtostderr: true,
+            ..Default::default()
+        })
+        .unwrap();
     build_info::build_info!(fn bi);
     info!("Build Info: {}", shared::build_version(bi));
     let allowed_origins = AllowedOrigins::all();
@@ -196,7 +328,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .mount(
             shared::urls::MOUNT_POINT,
             routes![
+                create_news_db,
+                reindex_news_db,
+                search_news_db,
                 original,
+                refresh,
                 show_pretty,
                 show,
                 graphql_query,
@@ -211,26 +347,33 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .attach(AdHoc::config::<Config>());
 
     let config: Config = rkt.figment().extract()?;
+    info!("Config:\n{config:#?}");
     if !std::fs::exists(&config.slurp_cache_path)? {
         info!("Creating slurp cache @ '{}'", &config.slurp_cache_path);
         std::fs::create_dir_all(&config.slurp_cache_path)?;
     }
     let pool = PgPool::connect(&config.newsreader_database_url).await?;
-    sqlx::migrate!("./migrations").run(&pool).await?;
+    let tantivy_newsreader_index = match Index::open_in_dir(&config.newsreader_tantivy_db_path) {
-    #[cfg(feature = "tantivy")]
+        Ok(idx) => idx,
-    let tantivy_conn = TantivyConnection::new(&config.newsreader_tantivy_db_path)?;
+        Err(_) => {
-
+            create_news_db_impl(&config)?;
+            Index::open_in_dir(&config.newsreader_tantivy_db_path)?
+        }
+    };
+    let tantivy_newsreader_reader = tantivy_newsreader_index.reader()?;
     let schema = Schema::build(QueryRoot, Mutation, EmptySubscription)
         .data(Notmuch::default())
         .data(config)
-        .data(pool.clone());
+        .data(pool.clone())
+        .extension(async_graphql::extensions::Logger)
+        .finish();
 
-    #[cfg(feature = "tantivy")]
+    let rkt = rkt
-    let schema = schema.data(tantivy_conn);
+        .manage(schema)
-
+        .manage(pool)
-    let schema = schema.extension(extensions::Logger).finish();
+        .manage(Notmuch::default())
-
+        .manage(tantivy_newsreader_index)
-    let rkt = rkt.manage(schema).manage(pool).manage(Notmuch::default());
+        .manage(tantivy_newsreader_reader);
     //.manage(Notmuch::with_config("../notmuch/testdata/notmuch.config"))
 
     rkt.launch().await?;

server/src/config.rs

@@ -1,7 +1,23 @@
-use serde::Deserialize;
+use std::{collections::HashMap, fmt::Display, str::FromStr};
-#[derive(Deserialize)]
+
+use scraper::Selector;
+use serde::{de, Deserialize, Deserializer};
+
+#[derive(Debug)]
+pub struct DeSelector(pub Selector);
+impl<'de> Deserialize<'de> for DeSelector {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        Ok(DeSelector(Selector::parse(&s).map_err(de::Error::custom)?))
+    }
+}
+#[derive(Debug, Deserialize)]
 pub struct Config {
     pub newsreader_database_url: String,
     pub newsreader_tantivy_db_path: String,
     pub slurp_cache_path: String,
+    pub slurp_site_selectors: HashMap<String, Vec<DeSelector>>,
 }

server/src/error.rs

@@ -1,8 +1,8 @@
 use std::{convert::Infallible, str::Utf8Error, string::FromUtf8Error};
 
 use mailparse::MailParseError;
-#[cfg(feature = "tantivy")]
+use tantivy::TantivyError;
-use tantivy::{query::QueryParserError, TantivyError};
+use tantivy::query::QueryParserError;
 use thiserror::Error;
 
 use crate::TransformError;
@@ -31,10 +31,8 @@ pub enum ServerError {
     StringError(String),
     #[error("invalid url: {0}")]
     UrlParseError(#[from] url::ParseError),
-    #[cfg(feature = "tantivy")]
     #[error("tantivy error: {0}")]
     TantivyError(#[from] TantivyError),
-    #[cfg(feature = "tantivy")]
     #[error("tantivy query parse error: {0}")]
     QueryParseError(#[from] QueryParserError),
     #[error("impossible: {0}")]

server/src/graphql.rs

@@ -1,5 +1,3 @@
-use std::{fmt, str::FromStr};
-
 use async_graphql::{
     connection::{self, Connection, Edge, OpaqueCursor},
     Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
@@ -9,11 +7,7 @@ use log::info;
 use notmuch::Notmuch;
 use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPool;
-use tokio::join;
-use tracing::instrument;
 
-#[cfg(feature = "tantivy")]
-use crate::tantivy::TantivyConnection;
 use crate::{config::Config, newsreader, nm, Query};
 
 /// # Number of seconds since the Epoch
@@ -22,26 +16,6 @@ pub type UnixTime = isize;
 /// # Thread ID, sans "thread:"
 pub type ThreadId = String;
 
-#[derive(Debug, Enum, Copy, Clone, Eq, PartialEq)]
-pub enum Corpus {
-    Notmuch,
-    Newsreader,
-    Tantivy,
-}
-
-impl FromStr for Corpus {
-    type Err = String;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(match s {
-            "notmuch" => Corpus::Notmuch,
-            "newsreader" => Corpus::Newsreader,
-            "tantivy" => Corpus::Tantivy,
-            s => return Err(format!("unknown corpus: '{s}'")),
-        })
-    }
-}
-
-// TODO: add is_read field and remove all use of 'tag:unread'
 #[derive(Debug, SimpleObject)]
 pub struct ThreadSummary {
     pub thread: ThreadId,
@@ -56,7 +30,6 @@ pub struct ThreadSummary {
     pub authors: String,
     pub subject: String,
     pub tags: Vec<String>,
-    pub corpus: Corpus,
 }
 
 #[derive(Debug, Union)]
@@ -237,19 +210,6 @@ impl Body {
 pub struct Email {
     pub name: Option<String>,
     pub addr: Option<String>,
-    pub photo_url: Option<String>,
-}
-
-impl fmt::Display for Email {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        match (&self.name, &self.addr) {
-            (Some(name), Some(addr)) => write!(f, "{name} <{addr}>")?,
-            (Some(name), None) => write!(f, "{name}")?,
-            (None, Some(addr)) => write!(f, "{addr}")?,
-            (None, None) => write!(f, "<UNKNOWN>")?,
-        }
-        Ok(())
-    }
 }
 
 #[derive(SimpleObject)]
@@ -264,17 +224,6 @@ pub struct Tag {
 struct SearchCursor {
     newsreader_offset: i32,
     notmuch_offset: i32,
-    #[cfg(feature = "tantivy")]
-    tantivy_offset: i32,
-}
-
-fn request_id() -> String {
-    let now = std::time::SystemTime::now();
-    let nanos = now
-        .duration_since(std::time::SystemTime::UNIX_EPOCH)
-        .unwrap_or_default()
-        .as_nanos();
-    format!("{nanos:x}")
 }
 
 pub struct QueryRoot;
@@ -284,31 +233,18 @@ impl QueryRoot {
         build_info::build_info!(fn bi);
         Ok(shared::build_version(bi))
     }
-    #[instrument(skip_all, fields(query=query))]
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
     async fn count<'ctx>(&self, ctx: &Context<'ctx>, query: String) -> Result<usize, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
-        #[cfg(feature = "tantivy")]
-        let tantivy = ctx.data_unchecked::<TantivyConnection>();
 
         let newsreader_query: Query = query.parse()?;
 
         let newsreader_count = newsreader::count(pool, &newsreader_query).await?;
-        let notmuch_count = nm::count(nm, &newsreader_query).await?;
+        let notmuch_count = nm::count(nm, &newsreader_query.to_notmuch()).await?;
-        #[cfg(feature = "tantivy")]
+        info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count}");
-        let tantivy_count = tantivy.count(&newsreader_query).await?;
+        Ok(newsreader_count + notmuch_count)
-        #[cfg(not(feature = "tantivy"))]
-        let tantivy_count = 0;
-
-        let total = newsreader_count + notmuch_count + tantivy_count;
-        info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count} tantivy count {tantivy_count} total {total}");
-        Ok(total)
     }
 
-    // TODO: this function doesn't get parallelism, possibly because notmuch is sync and blocks,
-    // rewrite that with tokio::process:Command
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
     async fn search<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -318,12 +254,15 @@ impl QueryRoot {
         last: Option<i32>,
         query: String,
     ) -> Result<Connection<OpaqueCursor<SearchCursor>, ThreadSummary>, Error> {
+        // TODO: add keywords to limit search to one corpus, i.e. is:news or is:mail
         info!("search({after:?} {before:?} {first:?} {last:?} {query:?})",);
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
-        #[cfg(feature = "tantivy")]
-        let tantivy = ctx.data_unchecked::<TantivyConnection>();
 
+        enum ThreadSummaryCursor {
+            Newsreader(i32, ThreadSummary),
+            Notmuch(i32, ThreadSummary),
+        }
         Ok(connection::query(
             after,
             before,
@@ -334,79 +273,64 @@ impl QueryRoot {
              first: Option<usize>,
              last: Option<usize>| async move {
                 info!(
-                    "search(after {:?} before {:?} first {first:?} last {last:?} query: {query:?})",
+                    "search({:?} {:?} {first:?} {last:?} {query:?})",
                     after.as_ref().map(|v| &v.0),
                     before.as_ref().map(|v| &v.0)
                 );
                 let newsreader_after = after.as_ref().map(|sc| sc.newsreader_offset);
                 let notmuch_after = after.as_ref().map(|sc| sc.notmuch_offset);
-                #[cfg(feature = "tantivy")]
-                let tantivy_after = after.as_ref().map(|sc| sc.tantivy_offset);
-
                 let newsreader_before = before.as_ref().map(|sc| sc.newsreader_offset);
                 let notmuch_before = before.as_ref().map(|sc| sc.notmuch_offset);
-                #[cfg(feature = "tantivy")]
-                let tantivy_before = before.as_ref().map(|sc| sc.tantivy_offset);
-                let first = first.map(|v| v as i32);
-                let last = last.map(|v| v as i32);
 
-                let query: Query = query.parse()?;
+                let newsreader_query: Query = query.parse()?;
-                info!("newsreader_query {query:?}");
+                info!("newsreader_query {newsreader_query:?}");
+                let newsreader_results = if newsreader_query.is_newsreader {
+                    newsreader::search(
+                        pool,
+                        newsreader_after,
+                        newsreader_before,
+                        first.map(|v| v as i32),
+                        last.map(|v| v as i32),
+                        &newsreader_query,
+                    )
+                    .await?
+                    .into_iter()
+                    .map(|(cur, ts)| ThreadSummaryCursor::Newsreader(cur, ts))
+                    .collect()
+                } else {
+                    Vec::new()
+                };
 
-                let newsreader_fut = newsreader_search(
+                let notmuch_results = if newsreader_query.is_notmuch {
-                    pool,
+                    nm::search(
-                    newsreader_after,
+                        nm,
-                    newsreader_before,
+                        notmuch_after,
-                    first,
+                        notmuch_before,
-                    last,
+                        first.map(|v| v as i32),
-                    &query,
+                        last.map(|v| v as i32),
-                );
+                        newsreader_query.to_notmuch(),
-                let notmuch_fut =
+                    )
-                    notmuch_search(nm, notmuch_after, notmuch_before, first, last, &query);
+                    .await?
-                #[cfg(feature = "tantivy")]
+                    .into_iter()
-                let tantivy_fut = tantivy_search(
+                    .map(|(cur, ts)| ThreadSummaryCursor::Notmuch(cur, ts))
-                    tantivy,
+                    .collect()
-                    pool,
+                } else {
-                    tantivy_after,
+                    Vec::new()
-                    tantivy_before,
+                };
-                    first,
-                    last,
-                    &query,
-                );
-                #[cfg(not(feature = "tantivy"))]
-                let tantivy_fut =
-                    async { Ok::<Vec<ThreadSummaryCursor>, async_graphql::Error>(Vec::new()) };
-
-                let (newsreader_results, notmuch_results, tantivy_results) =
-                    join!(newsreader_fut, notmuch_fut, tantivy_fut);
-
-                let newsreader_results = newsreader_results?;
-                let notmuch_results = notmuch_results?;
-                let tantivy_results = tantivy_results?;
-                info!(
-                    "newsreader_results ({}) notmuch_results ({}) tantivy_results ({})",
-                    newsreader_results.len(),
-                    notmuch_results.len(),
-                    tantivy_results.len()
-                );
 
                 let mut results: Vec<_> = newsreader_results
                     .into_iter()
                     .chain(notmuch_results)
-                    .chain(tantivy_results)
                     .collect();
 
                 // The leading '-' is to reverse sort
                 results.sort_by_key(|item| match item {
                     ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
                     ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
-                    #[cfg(feature = "tantivy")]
-                    ThreadSummaryCursor::Tantivy(_, ts) => -ts.timestamp,
                 });
 
                 let mut has_next_page = before.is_some();
                 if let Some(first) = first {
-                    let first = first as usize;
                     if results.len() > first {
                         has_next_page = true;
                         results.truncate(first);
@@ -415,7 +339,6 @@ impl QueryRoot {
 
                 let mut has_previous_page = after.is_some();
                 if let Some(last) = last {
-                    let last = last as usize;
                     if results.len() > last {
                         has_previous_page = true;
                         results.truncate(last);
@@ -423,17 +346,8 @@ impl QueryRoot {
                 }
 
                 let mut connection = Connection::new(has_previous_page, has_next_page);
-                // Set starting offset as the value from cursor to preserve state if no results from a corpus survived the truncation
+                let mut newsreader_offset = 0;
-                let mut newsreader_offset =
+                let mut notmuch_offset = 0;
-                    after.as_ref().map(|sc| sc.newsreader_offset).unwrap_or(0);
-                let mut notmuch_offset = after.as_ref().map(|sc| sc.notmuch_offset).unwrap_or(0);
-                #[cfg(feature = "tantivy")]
-                let tantivy_offset = after.as_ref().map(|sc| sc.tantivy_offset).unwrap_or(0);
-
-                info!(
-                    "newsreader_offset ({}) notmuch_offset ({})",
-                    newsreader_offset, notmuch_offset,
-                );
 
                 connection.edges.extend(results.into_iter().map(|item| {
                     let thread_summary;
@@ -446,17 +360,10 @@ impl QueryRoot {
                             thread_summary = ts;
                             notmuch_offset = offset;
                         }
-                        #[cfg(feature = "tantivy")]
-                        ThreadSummaryCursor::Tantivy(offset, ts) => {
-                            thread_summary = ts;
-                            tantivy_offset = offset;
-                        }
                     }
                     let cur = OpaqueCursor(SearchCursor {
                         newsreader_offset,
                         notmuch_offset,
-                        #[cfg(feature = "tantivy")]
-                        tantivy_offset,
                     });
                     Edge::new(cur, thread_summary)
                 }));
@@ -466,7 +373,6 @@ impl QueryRoot {
         .await?)
     }
 
-    #[instrument(skip_all, fields(request_id=request_id()))]
     async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -475,7 +381,6 @@ impl QueryRoot {
         tags.append(&mut nm::tags(nm, needs_unread)?);
         Ok(tags)
     }
-    #[instrument(skip_all, fields(thread_id=thread_id, request_id=request_id()))]
     async fn thread<'ctx>(&self, ctx: &Context<'ctx>, thread_id: String) -> Result<Thread, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -486,73 +391,18 @@ impl QueryRoot {
             .field("body")
             .field("contentTree")
             .exists();
+        // TODO: look at thread_id and conditionally load newsreader
         if newsreader::is_newsreader_thread(&thread_id) {
             Ok(newsreader::thread(config, pool, thread_id).await?)
         } else {
-            Ok(nm::thread(nm, pool, thread_id, debug_content_tree).await?)
+            Ok(nm::thread(nm, thread_id, debug_content_tree).await?)
         }
     }
 }
 
-#[derive(Debug)]
-enum ThreadSummaryCursor {
-    Newsreader(i32, ThreadSummary),
-    Notmuch(i32, ThreadSummary),
-    #[cfg(feature = "tantivy")]
-    Tantivy(i32, ThreadSummary),
-}
-async fn newsreader_search(
-    pool: &PgPool,
-    after: Option<i32>,
-    before: Option<i32>,
-    first: Option<i32>,
-    last: Option<i32>,
-    query: &Query,
-) -> Result<Vec<ThreadSummaryCursor>, async_graphql::Error> {
-    Ok(newsreader::search(pool, after, before, first, last, &query)
-        .await?
-        .into_iter()
-        .map(|(cur, ts)| ThreadSummaryCursor::Newsreader(cur, ts))
-        .collect())
-}
-
-async fn notmuch_search(
-    nm: &Notmuch,
-    after: Option<i32>,
-    before: Option<i32>,
-    first: Option<i32>,
-    last: Option<i32>,
-    query: &Query,
-) -> Result<Vec<ThreadSummaryCursor>, async_graphql::Error> {
-    Ok(nm::search(nm, after, before, first, last, &query)
-        .await?
-        .into_iter()
-        .map(|(cur, ts)| ThreadSummaryCursor::Notmuch(cur, ts))
-        .collect())
-}
-
-#[cfg(feature = "tantivy")]
-async fn tantivy_search(
-    tantivy: &TantivyConnection,
-    pool: &PgPool,
-    after: Option<i32>,
-    before: Option<i32>,
-    first: Option<i32>,
-    last: Option<i32>,
-    query: &Query,
-) -> Result<Vec<ThreadSummaryCursor>, async_graphql::Error> {
-    Ok(tantivy
-        .search(pool, after, before, first, last, &query)
-        .await?
-        .into_iter()
-        .map(|(cur, ts)| ThreadSummaryCursor::Tantivy(cur, ts))
-        .collect())
-}
-
 pub struct Mutation;
 #[Object]
 impl Mutation {
-    #[instrument(skip_all, fields(query=query, unread=unread, request_id=request_id()))]
     async fn set_read_status<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -561,17 +411,16 @@ impl Mutation {
     ) -> Result<bool, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
-        #[cfg(feature = "tantivy")]
-        let tantivy = ctx.data_unchecked::<TantivyConnection>();
 
-        let query: Query = query.parse()?;
+        for q in query.split_whitespace() {
-        newsreader::set_read_status(pool, &query, unread).await?;
+            if newsreader::is_newsreader_thread(&q) {
-        #[cfg(feature = "tantivy")]
+                newsreader::set_read_status(pool, &q, unread).await?;
-        tantivy.reindex_thread(pool, &query).await?;
+            } else {
-        nm::set_read_status(nm, &query, unread).await?;
+                nm::set_read_status(nm, q, unread).await?;
+            }
+        }
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
     async fn tag_add<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -583,7 +432,6 @@ impl Mutation {
         nm.tag_add(&tag, &query)?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
     async fn tag_remove<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -595,30 +443,6 @@ impl Mutation {
         nm.tag_remove(&tag, &query)?;
         Ok(true)
     }
-    /// Drop and recreate tantivy index. Warning this is slow
-    #[cfg(feature = "tantivy")]
-    async fn drop_and_load_index<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
-        let tantivy = ctx.data_unchecked::<TantivyConnection>();
-        let pool = ctx.data_unchecked::<PgPool>();
-
-        tantivy.drop_and_load_index()?;
-        tantivy.reindex_all(pool).await?;
-
-        Ok(true)
-    }
-    #[instrument(skip_all, fields(request_id=request_id()))]
-    async fn refresh<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
-        let nm = ctx.data_unchecked::<Notmuch>();
-        info!("{}", String::from_utf8_lossy(&nm.new()?));
-        #[cfg(feature = "tantivy")]
-        {
-            let tantivy = ctx.data_unchecked::<TantivyConnection>();
-            let pool = ctx.data_unchecked::<PgPool>();
-            // TODO: parallelize
-            tantivy.refresh(pool).await?;
-        }
-        Ok(true)
-    }
 }
 
 pub type GraphqlSchema = Schema<QueryRoot, Mutation, EmptySubscription>;

server/src/lib.rs

@@ -1,13 +1,10 @@
 pub mod config;
 pub mod error;
 pub mod graphql;
-pub mod mail;
 pub mod newsreader;
 pub mod nm;
-#[cfg(feature = "tantivy")]
-pub mod tantivy;
 
-use std::{collections::HashMap, convert::Infallible, fmt, str::FromStr, sync::Arc};
+use std::{collections::HashMap, convert::Infallible, str::FromStr, sync::Arc};
 
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
@@ -20,21 +17,15 @@ use lol_html::{
 };
 use maplit::{hashmap, hashset};
 use scraper::{Html, Selector};
-use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
 use tokio::sync::Mutex;
 use url::Url;
 
 use crate::{
-    error::ServerError,
+    config::DeSelector,
-    graphql::{Corpus, ThreadSummary},
+    newsreader::{extract_thread_id, is_newsreader_thread},
-    newsreader::is_newsreader_thread,
-    nm::is_notmuch_thread_or_id,
 };
 
-const NEWSREADER_TAG_PREFIX: &'static str = "News/";
-const NEWSREADER_THREAD_PREFIX: &'static str = "news:";
-
 // TODO: figure out how to use Cow
 #[async_trait]
 trait Transformer: Send + Sync {
@@ -110,48 +101,6 @@ impl Transformer for StripHtml {
     }
 }
 
-struct InlineRemoteStyle<'a> {
-    base_url: &'a Option<Url>,
-}
-
-#[async_trait]
-impl<'a> Transformer for InlineRemoteStyle<'a> {
-    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
-        let css = concat!(
-            "/* chrome-default.css */\n",
-            include_str!("chrome-default.css"),
-            "\n/* mvp.css */\n",
-            include_str!("mvp.css"),
-            "\n/* Xinu Specific overrides */\n",
-            include_str!("custom.css"),
-        );
-        let inline_opts = InlineOptions {
-            //inline_style_tags: true,
-            //keep_style_tags: false,
-            //keep_link_tags: true,
-            base_url: self.base_url.clone(),
-            //load_remote_stylesheets: true,
-            //preallocate_node_capacity: 32,
-            ..InlineOptions::default()
-        };
-
-        //info!("HTML:\n{html}");
-        info!("base_url: {:#?}", self.base_url);
-        Ok(
-            match CSSInliner::options()
-                .base_url(self.base_url.clone())
-                .build()
-                .inline(&html)
-            {
-                Ok(inlined_html) => inlined_html,
-                Err(err) => {
-                    error!("failed to inline remote CSS: {err}");
-                    html.to_string()
-                }
-            },
-        )
-    }
-}
 struct InlineStyle;
 
 #[async_trait]
@@ -269,14 +218,13 @@ impl Transformer for AddOutlink {
     }
 }
 
-struct SlurpContents {
+struct SlurpContents<'h> {
     cacher: Arc<Mutex<FilesystemCacher>>,
-    inline_css: bool,
+    site_selectors: &'h HashMap<String, Vec<DeSelector>>,
-    site_selectors: HashMap<String, Vec<Selector>>,
 }
 
-impl SlurpContents {
+impl<'h> SlurpContents<'h> {
-    fn get_selectors(&self, link: &Url) -> Option<&[Selector]> {
+    fn get_selectors(&self, link: &Url) -> Option<&[DeSelector]> {
         for (host, selector) in self.site_selectors.iter() {
             if link.host_str().map(|h| h.contains(host)).unwrap_or(false) {
                 return Some(&selector);
@@ -287,7 +235,7 @@ impl SlurpContents {
 }
 
 #[async_trait]
-impl Transformer for SlurpContents {
+impl<'h> Transformer for SlurpContents<'h> {
     fn should_run(&self, link: &Option<Url>, _: &str) -> bool {
         if let Some(link) = link {
             return self.get_selectors(link).is_some();
@@ -301,7 +249,7 @@ impl Transformer for SlurpContents {
         let Some(selectors) = self.get_selectors(&link) else {
             return Ok(html.to_string());
         };
-        let cacher = self.cacher.lock().await;
+        let mut cacher = self.cacher.lock().await;
         let body = if let Some(body) = cacher.get(link.as_str()) {
             info!("cache hit for {link}");
             String::from_utf8_lossy(&body).to_string()
@@ -310,47 +258,17 @@ impl Transformer for SlurpContents {
             cacher.set(link.as_str(), body.as_bytes());
             body
         };
-        let body = Arc::new(body);
-        let base_url = Some(link.clone());
-        let body = if self.inline_css {
-            let inner_body = Arc::clone(&body);
-            let res = tokio::task::spawn_blocking(move || {
-                let res = CSSInliner::options()
-                    .base_url(base_url)
-                    .build()
-                    .inline(&inner_body);
-
-                match res {
-                    Ok(inlined_html) => inlined_html,
-                    Err(err) => {
-                        error!("failed to inline remote CSS: {err}");
-                        Arc::into_inner(inner_body).expect("failed to take body out of Arc")
-                    }
-                }
-            })
-            .await;
-            match res {
-                Ok(inlined_html) => inlined_html,
-                Err(err) => {
-                    error!("failed to spawn inline remote CSS: {err}");
-                    Arc::into_inner(body).expect("failed to take body out of Arc")
-                }
-            }
-        } else {
-            Arc::into_inner(body).expect("failed to take body out of Arc")
-        };
-
         let doc = Html::parse_document(&body);
 
         let mut results = Vec::new();
         for selector in selectors {
-            for frag in doc.select(&selector) {
+            for frag in doc.select(&selector.0) {
                 results.push(frag.html())
                 // TODO: figure out how to warn if there were no hits
                 //warn!("couldn't find '{:?}' in {}", selector, link);
             }
         }
-        Ok(results.join("<br>"))
+        Ok(results.join(""))
     }
 }
 
@@ -680,47 +598,14 @@ fn compute_offset_limit(
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct Query {
     pub unread_only: bool,
-    pub tags: Vec<String>,
+    pub tag: Option<String>,
-    pub uids: Vec<String>,
+    pub uid: Option<String>,
     pub remainder: Vec<String>,
     pub is_notmuch: bool,
     pub is_newsreader: bool,
-    pub is_tantivy: bool,
-    pub corpus: Option<Corpus>,
-}
-
-impl fmt::Display for Query {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        if self.unread_only {
-            write!(f, "is:unread ")?;
-        }
-        for tag in &self.tags {
-            write!(f, "tag:{tag} ")?;
-        }
-        for uid in &self.uids {
-            write!(f, "id:{uid} ")?;
-        }
-        if self.is_notmuch {
-            write!(f, "is:mail ")?;
-        }
-        if self.is_newsreader {
-            write!(f, "is:newsreader ")?;
-        }
-        if self.is_newsreader {
-            write!(f, "is:news ")?;
-        }
-        match self.corpus {
-            Some(c) => write!(f, "corpus:{c:?}")?,
-            _ => (),
-        }
-        for rem in &self.remainder {
-            write!(f, "{rem} ")?;
-        }
-        Ok(())
-    }
 }
 
 impl Query {
@@ -735,10 +620,10 @@ impl Query {
         if self.unread_only {
             parts.push("is:unread".to_string());
         }
-        for tag in &self.tags {
+        if let Some(site) = &self.tag {
-            parts.push(format!("tag:{tag}"));
+            parts.push(format!("tag:{site}"));
         }
-        for uid in &self.uids {
+        if let Some(uid) = &self.uid {
             parts.push(uid.clone());
         }
         parts.extend(self.remainder.clone());
@@ -750,109 +635,44 @@ impl FromStr for Query {
     type Err = Infallible;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut unread_only = false;
-        let mut tags = Vec::new();
+        let mut tag = None;
-        let mut uids = Vec::new();
+        let mut uid = None;
         let mut remainder = Vec::new();
         let mut is_notmuch = false;
         let mut is_newsreader = false;
-        let mut is_tantivy = false;
-        let mut corpus = None;
         for word in s.split_whitespace() {
             if word == "is:unread" {
                 unread_only = true
             } else if word.starts_with("tag:") {
-                tags.push(word["tag:".len()..].to_string());
+                tag = Some(word["tag:".len()..].to_string())
-
                 /*
                             } else if word.starts_with("tag:") {
                                 // Any tag that doesn't match site_prefix should explicitly set the site to something not in the
                                 // database
                                 site = Some(NON_EXISTENT_SITE_NAME.to_string());
                 */
-            } else if word.starts_with("corpus:") {
-                let c = word["corpus:".len()..].to_string();
-                corpus = c.parse::<Corpus>().map(|c| Some(c)).unwrap_or_else(|e| {
-                    warn!("Error parsing corpus '{c}': {e:?}");
-                    None
-                });
             } else if is_newsreader_thread(word) {
-                uids.push(word.to_string());
+                uid = Some(extract_thread_id(word).to_string())
-            } else if is_notmuch_thread_or_id(word) {
-                uids.push(word.to_string());
             } else if word == "is:mail" || word == "is:email" || word == "is:notmuch" {
                 is_notmuch = true;
-            } else if word == "is:news" {
+            } else if word == "is:news" || word == "is:newsreader" {
-                is_newsreader = true;
-            } else if word == "is:newsreader" {
                 is_newsreader = true;
             } else {
                 remainder.push(word.to_string());
             }
         }
         // If we don't see any explicit filters for a corpus, flip them all on
-        if corpus.is_none() && !(is_notmuch || is_tantivy || is_newsreader) {
+        if !(is_notmuch || is_newsreader) {
-            is_notmuch = true;
             is_newsreader = true;
-            is_tantivy = true;
+            is_notmuch = true;
         }
         Ok(Query {
             unread_only,
-            tags,
+            tag,
-            uids,
+            uid,
             remainder,
             is_notmuch,
             is_newsreader,
-            is_tantivy,
-            corpus,
         })
     }
 }
-pub struct ThreadSummaryRecord {
-    pub site: Option<String>,
-    pub date: Option<PrimitiveDateTime>,
-    pub is_read: Option<bool>,
-    pub title: Option<String>,
-    pub uid: String,
-    pub name: Option<String>,
-    pub corpus: Corpus,
-}
-
-async fn thread_summary_from_row(r: ThreadSummaryRecord) -> ThreadSummary {
-    let site = r.site.unwrap_or("UNKOWN TAG".to_string());
-    let mut tags = vec![format!("{NEWSREADER_TAG_PREFIX}{site}")];
-    if !r.is_read.unwrap_or(true) {
-        tags.push("unread".to_string());
-    };
-    let mut title = r.title.unwrap_or("NO TITLE".to_string());
-    title = clean_title(&title).await.expect("failed to clean title");
-    ThreadSummary {
-        thread: format!("{NEWSREADER_THREAD_PREFIX}{}", r.uid),
-        timestamp: r
-            .date
-            .expect("post missing date")
-            .assume_utc()
-            .unix_timestamp() as isize,
-        date_relative: format!("{:?}", r.date),
-        //date_relative: "TODO date_relative".to_string(),
-        matched: 0,
-        total: 1,
-        authors: r.name.unwrap_or_else(|| site.clone()),
-        subject: title,
-        tags,
-        corpus: r.corpus,
-    }
-}
-async fn clean_title(title: &str) -> Result<String, ServerError> {
-    // Make title HTML so html parsers work
-    let mut title = format!("<html>{title}</html>");
-    let title_tranformers: Vec<Box<dyn Transformer>> =
-        vec![Box::new(EscapeHtml), Box::new(StripHtml)];
-    // Make title HTML so html parsers work
-    title = format!("<html>{title}</html>");
-    for t in title_tranformers.iter() {
-        if t.should_run(&None, &title) {
-            title = t.transform(&None, &title).await?;
-        }
-    }
-    Ok(title)
-}

server/src/mail.rs

@@ -1,113 +0,0 @@
-use std::{fs::File, io::Read};
-
-use mailparse::{
-    addrparse_header, dateparse, parse_mail, MailHeaderMap, MailParseError, ParsedMail,
-};
-use sqlx::postgres::PgPool;
-use thiserror::Error;
-use tracing::info;
-
-#[derive(Error, Debug)]
-pub enum MailError {
-    #[error("missing from header")]
-    MissingFrom,
-    #[error("missing from header display name")]
-    MissingFromDisplayName,
-    #[error("missing subject header")]
-    MissingSubject,
-    #[error("missing html part")]
-    MissingHtmlPart,
-    #[error("missing message ID")]
-    MissingMessageId,
-    #[error("missing date")]
-    MissingDate,
-    #[error("DB error {0}")]
-    SqlxError(#[from] sqlx::Error),
-    #[error("IO error {0}")]
-    IOError(#[from] std::io::Error),
-    #[error("mail parse error {0}")]
-    MailParseError(#[from] MailParseError),
-}
-
-pub async fn read_mail_to_db(pool: &PgPool, path: &str) -> Result<(), MailError> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    file.read_to_end(&mut buffer)?;
-    let m = parse_mail(&buffer)?;
-
-    let subject = m
-        .headers
-        .get_first_value("subject")
-        .ok_or(MailError::MissingSubject)?;
-
-    let from = addrparse_header(
-        m.headers
-            .get_first_header("from")
-            .ok_or(MailError::MissingFrom)?,
-    )?;
-    let from = from.extract_single_info().ok_or(MailError::MissingFrom)?;
-    let name = from.display_name.ok_or(MailError::MissingFromDisplayName)?;
-    let slug = name.to_lowercase().replace(' ', "-");
-    let url = from.addr;
-    let message_id = m
-        .headers
-        .get_first_value("Message-ID")
-        .ok_or(MailError::MissingMessageId)?;
-    let uid = &message_id;
-    let feed_id = find_feed(&pool, &name, &slug, &url).await?;
-    let date = dateparse(
-        &m.headers
-            .get_first_value("Date")
-            .ok_or(MailError::MissingDate)?,
-    )?;
-
-    println!("Feed: {feed_id} Subject: {}", subject);
-
-    if let Some(_m) = first_html(&m) {
-        info!("add email {slug} {subject} {message_id} {date} {uid} {url}");
-    } else {
-        return Err(MailError::MissingHtmlPart.into());
-    }
-
-    Ok(())
-}
-fn first_html<'m>(m: &'m ParsedMail<'m>) -> Option<&'m ParsedMail<'m>> {
-    for ele in m.parts() {
-        if ele.ctype.mimetype == "text/html" {
-            return Some(ele);
-        }
-    }
-    None
-}
-async fn find_feed(pool: &PgPool, name: &str, slug: &str, url: &str) -> Result<i32, MailError> {
-    match sqlx::query!(
-        r#"
-SELECT id
-FROM feed
-WHERE slug = $1
-        "#,
-        slug
-    )
-    .fetch_one(pool)
-    .await
-    {
-        Err(sqlx::Error::RowNotFound) => {
-            let rec = sqlx::query!(
-                r#"
-INSERT INTO feed ( name, slug, url, homepage, selector )
-VALUES ( $1, $2, $3, '', '' )
-RETURNING id
-        "#,
-                name,
-                slug,
-                url
-            )
-            .fetch_one(pool)
-            .await?;
-
-            return Ok(rec.id);
-        }
-        Ok(rec) => return Ok(rec.id),
-        Err(e) => return Err(e.into()),
-    };
-}

server/src/newsreader.rs

@@ -7,79 +7,53 @@ use scraper::Selector;
 use shared::compute_color;
 use sqlx::postgres::PgPool;
 use tokio::sync::Mutex;
-use tracing::instrument;
 use url::Url;
 
 use crate::{
-    clean_title, compute_offset_limit,
+    compute_offset_limit,
     config::Config,
     error::ServerError,
-    graphql::{Corpus, NewsPost, Tag, Thread, ThreadSummary},
+    graphql::{NewsPost, Tag, Thread, ThreadSummary},
-    thread_summary_from_row, AddOutlink, EscapeHtml, FrameImages, InlineRemoteStyle, Query,
+    AddOutlink, EscapeHtml, FrameImages, InlineStyle, Query, SanitizeHtml, SlurpContents,
-    SanitizeHtml, SlurpContents, ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX,
+    StripHtml, Transformer,
-    NEWSREADER_THREAD_PREFIX,
 };
 
-pub fn is_newsreader_query(query: &Query) -> bool {
+const TAG_PREFIX: &'static str = "News/";
-    query.is_newsreader || query.corpus == Some(Corpus::Newsreader)
+const THREAD_PREFIX: &'static str = "news:";
+
+pub fn is_newsreader_search(query: &str) -> bool {
+    query.contains(TAG_PREFIX)
 }
 
 pub fn is_newsreader_thread(query: &str) -> bool {
-    query.starts_with(NEWSREADER_THREAD_PREFIX)
+    query.starts_with(THREAD_PREFIX)
 }
 
 pub fn extract_thread_id(query: &str) -> &str {
-    if query.starts_with(NEWSREADER_THREAD_PREFIX) {
+    &query[THREAD_PREFIX.len()..]
-        &query[NEWSREADER_THREAD_PREFIX.len()..]
-    } else {
-        query
-    }
 }
 
 pub fn extract_site(tag: &str) -> &str {
-    &tag[NEWSREADER_TAG_PREFIX.len()..]
+    &tag[TAG_PREFIX.len()..]
 }
 
 pub fn make_news_tag(tag: &str) -> String {
-    format!("tag:{NEWSREADER_TAG_PREFIX}{tag}")
+    format!("tag:{TAG_PREFIX}{tag}")
 }
 
-fn site_from_tags(tags: &[String]) -> Option<String> {
-    for t in tags {
-        if t.starts_with(NEWSREADER_TAG_PREFIX) {
-            return Some(extract_site(t).to_string());
-        }
-    }
-    None
-}
-
-#[instrument(name = "newsreader::count", skip_all, fields(query=%query))]
 pub async fn count(pool: &PgPool, query: &Query) -> Result<usize, ServerError> {
-    if !is_newsreader_query(query) {
+    if !query.remainder.is_empty() {
-        return Ok(0);
+        // TODO: handle full text search against all sites, for now, early return if search words
-    }
+        // are specified.
-    let site = site_from_tags(&query.tags);
-    if !query.tags.is_empty() && site.is_none() {
-        // Newsreader can only handle all sites read/unread queries, anything with a non-site tag
-        // isn't supported
         return Ok(0);
     }
 
-    let search_term = query.remainder.join(" ");
+    let row = sqlx::query_file!("sql/count.sql", query.tag, query.unread_only)
-    let search_term = search_term.trim();
-    let search_term = if search_term.is_empty() {
-        None
-    } else {
-        Some(search_term)
-    };
-    // TODO: add support for looking for search_term in title and site
-    let row = sqlx::query_file!("sql/count.sql", site, query.unread_only, search_term)
         .fetch_one(pool)
         .await?;
     Ok(row.count.unwrap_or(0).try_into().unwrap_or(0))
 }
 
-#[instrument(name = "newsreader::search", skip_all, fields(query=%query))]
 pub async fn search(
     pool: &PgPool,
     after: Option<i32>,
@@ -89,13 +63,9 @@ pub async fn search(
     query: &Query,
 ) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
     info!("search({after:?} {before:?} {first:?} {last:?} {query:?}");
-    if !is_newsreader_query(query) {
+    if !query.remainder.is_empty() {
-        return Ok(Vec::new());
+        // TODO: handle full text search against all sites, for now, early return if search words
-    }
+        // are specified.
-    let site = site_from_tags(&query.tags);
-    if !query.tags.is_empty() && site.is_none() {
-        // Newsreader can only handle all sites read/unread queries, anything with a non-site tag
-        // isn't supported
         return Ok(Vec::new());
     }
 
@@ -107,48 +77,53 @@ pub async fn search(
         limit = limit + 1;
     }
 
+    let site = query.tag.as_ref().map(|t| extract_site(&t).to_string());
     info!(
         "search offset {offset} limit {limit} site {site:?} unread_only {}",
         query.unread_only
     );
-    let search_term = query.remainder.join(" ");
-    let search_term = search_term.trim();
-    let search_term = if search_term.is_empty() {
-        None
-    } else {
-        Some(search_term)
-    };
 
-    // TODO: add support for looking for search_term in title and site
+    // TODO: further limit results to include query.remainder if set
     let rows = sqlx::query_file!(
         "sql/threads.sql",
         site,
         query.unread_only,
         offset as i64,
-        limit as i64,
+        limit as i64
-        search_term
     )
     .fetch_all(pool)
     .await?;
+
     let mut res = Vec::new();
     for (i, r) in rows.into_iter().enumerate() {
+        let site = r.site.unwrap_or("UNKOWN TAG".to_string());
+        let mut tags = vec![format!("{TAG_PREFIX}{site}")];
+        if !r.is_read.unwrap_or(true) {
+            tags.push("unread".to_string());
+        };
+        let mut title = r.title.unwrap_or("NO TITLE".to_string());
+        title = clean_title(&title).await.expect("failed to clean title");
         res.push((
             i as i32 + offset,
-            thread_summary_from_row(ThreadSummaryRecord {
+            ThreadSummary {
-                site: r.site,
+                thread: format!("{THREAD_PREFIX}{}", r.uid),
-                date: r.date,
+                timestamp: r
-                is_read: r.is_read,
+                    .date
-                title: r.title,
+                    .expect("post missing date")
-                uid: r.uid,
+                    .assume_utc()
-                name: r.name,
+                    .unix_timestamp() as isize,
-                corpus: Corpus::Newsreader,
+                date_relative: "TODO date_relative".to_string(),
-            })
+                matched: 0,
-            .await,
+                total: 1,
+                authors: r.name.unwrap_or_else(|| site.clone()),
+                subject: title,
+                tags,
+            },
         ));
     }
     Ok(res)
 }
-#[instrument(name = "newsreader::tags", skip_all, fields(needs_unread=%_needs_unread))]
+
 pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     // TODO: optimize query by using needs_unread
     let tags = sqlx::query_file!("sql/tags.sql").fetch_all(pool).await?;
@@ -156,10 +131,7 @@ pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, Server
         .into_iter()
         .map(|tag| {
             let unread = tag.unread.unwrap_or(0).try_into().unwrap_or(0);
-            let name = format!(
+            let name = format!("{TAG_PREFIX}{}", tag.site.expect("tag must have site"));
-                "{NEWSREADER_TAG_PREFIX}{}",
-                tag.site.expect("tag must have site")
-            );
             let hex = compute_color(&name);
             Tag {
                 name,
@@ -172,15 +144,14 @@ pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, Server
     Ok(tags)
 }
 
-#[instrument(name = "newsreader::thread", skip_all, fields(thread_id=%thread_id))]
 pub async fn thread(
     config: &Config,
     pool: &PgPool,
     thread_id: String,
 ) -> Result<Thread, ServerError> {
     let id = thread_id
-        .strip_prefix(NEWSREADER_THREAD_PREFIX)
+        .strip_prefix(THREAD_PREFIX)
-        .expect("news thread doesn't start with '{NEWSREADER_THREAD_PREFIX}'")
+        .expect("news thread doesn't start with '{THREAD_PREFIX}'")
         .to_string();
 
     let r = sqlx::query_file!("sql/thread.sql", id)
@@ -189,76 +160,37 @@ pub async fn thread(
 
     let slug = r.site.unwrap_or("no-slug".to_string());
     let site = r.name.unwrap_or("NO SITE".to_string());
-    // TODO: remove the various places that have this as an Option
+    let default_homepage = "http://no-homepage";
-    let link = Some(Url::parse(&r.link)?);
+    let link = &r
+        .link
+        .as_ref()
+        .map(|h| {
+            if h.is_empty() {
+                default_homepage.to_string()
+            } else {
+                h.to_string()
+            }
+        })
+        .map(|h| Url::parse(&h).ok())
+        .flatten();
     let mut body = r.summary.unwrap_or("NO SUMMARY".to_string());
+    // TODO: add site specific cleanups. For example:
+    // * Grafana does <div class="image-wrapp"><img class="lazyload>"<img src="/media/...>"</img></div>
+    // * Some sites appear to be HTML encoded, unencode them, i.e. imperialviolent
     let cacher = Arc::new(Mutex::new(FilesystemCacher::new(&config.slurp_cache_path)?));
     let body_tranformers: Vec<Box<dyn Transformer>> = vec![
         Box::new(SlurpContents {
             cacher,
-            // TODO: make this true when bulma is finally removed
+            site_selectors: &config.slurp_site_selectors,
-            inline_css: false,
-            site_selectors: hashmap![
-                "atmeta.com".to_string() => vec![
-                    Selector::parse("div.entry-content").unwrap(),
-                ],
-                "blog.prusa3d.com".to_string() => vec![
-                    Selector::parse("article.content .post-block").unwrap(),
-                ],
-                "blog.cloudflare.com".to_string() => vec![
-                    Selector::parse(".author-lists .author-name-tooltip").unwrap(),
-                    Selector::parse(".post-full-content").unwrap()
-                ],
-                "blog.zsa.io".to_string() => vec![
-                    Selector::parse("section.blog-article").unwrap(),
-                ],
-                "engineering.fb.com".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "grafana.com".to_string() => vec![
-                    Selector::parse(".blog-content").unwrap(),
-                ],
-                "hackaday.com".to_string() => vec![
-                    Selector::parse("div.entry-featured-image").unwrap(),
-                    Selector::parse("div.entry-content").unwrap()
-                ],
-                "ingowald.blog".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "jvns.ca".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
-                "natwelch.com".to_string() => vec![
-                    Selector::parse("article div.prose").unwrap(),
-                ],
-                "rustacean-station.org".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "slashdot.org".to_string() => vec![
-                    Selector::parse("span.story-byline").unwrap(),
-                    Selector::parse("div.p").unwrap(),
-                ],
-                "trofi.github.io".to_string() => vec![
-                    Selector::parse("#content").unwrap(),
-                ],
-                "www.redox-os.org".to_string() => vec![
-                    Selector::parse("div.content").unwrap(),
-                ],
-                "www.smbc-comics.com".to_string() => vec![
-                    Selector::parse("img#cc-comic").unwrap(),
-                    Selector::parse("div#aftercomic img").unwrap(),
-                ],
-            ],
         }),
         Box::new(FrameImages),
         Box::new(AddOutlink),
-        // TODO: causes doubling of images in cloudflare blogs
+        Box::new(EscapeHtml),
-        //Box::new(EscapeHtml),
         Box::new(SanitizeHtml {
             cid_prefix: "",
             base_url: &link,
         }),
+        Box::new(InlineStyle),
     ];
     for t in body_tranformers.iter() {
         if t.should_run(&link, &body) {
@@ -286,25 +218,28 @@ pub async fn thread(
         timestamp,
     }))
 }
-#[instrument(name = "newsreader::set_read_status", skip_all, fields(query=%query,unread=%unread))]
 pub async fn set_read_status<'ctx>(
     pool: &PgPool,
-    query: &Query,
+    query: &str,
     unread: bool,
 ) -> Result<bool, ServerError> {
-    // TODO: make single query when query.uids.len() > 1
+    let query: Query = query.parse()?;
-    let uids: Vec<_> = query
+    sqlx::query_file!("sql/set_unread.sql", !unread, query.uid)
-        .uids
+        .execute(pool)
-        .iter()
+        .await?;
-        .filter(|uid| is_newsreader_thread(uid))
-        .map(
-            |uid| extract_thread_id(uid), // TODO strip prefix
-        )
-        .collect();
-    for uid in uids {
-        sqlx::query_file!("sql/set_unread.sql", !unread, uid)
-            .execute(pool)
-            .await?;
-    }
     Ok(true)
 }
+async fn clean_title(title: &str) -> Result<String, ServerError> {
+    // Make title HTML so html parsers work
+    let mut title = format!("<html>{title}</html>");
+    let title_tranformers: Vec<Box<dyn Transformer>> =
+        vec![Box::new(EscapeHtml), Box::new(StripHtml)];
+    // Make title HTML so html parsers work
+    title = format!("<html>{title}</html>");
+    for t in title_tranformers.iter() {
+        if t.should_run(&None, &title) {
+            title = t.transform(&None, &title).await?;
+        }
+    }
+    Ok(title)
+}

server/src/nm.rs

@@ -6,42 +6,30 @@ use std::{
 };
 
 use log::{error, info, warn};
-use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
+use mailparse::{parse_mail, MailHeader, MailHeaderMap, ParsedMail};
 use memmap::MmapOptions;
 use notmuch::Notmuch;
-use sqlx::PgPool;
-use tracing::instrument;
 
 use crate::{
     compute_offset_limit,
     error::ServerError,
     graphql::{
-        Attachment, Body, Corpus, DispositionType, Email, EmailThread, Header, Html, Message,
+        Attachment, Body, DispositionType, Email, EmailThread, Header, Html, Message, PlainText,
-        PlainText, Tag, Thread, ThreadSummary, UnhandledContentType,
+        Tag, Thread, ThreadSummary, UnhandledContentType,
     },
-    linkify_html, InlineStyle, Query, SanitizeHtml, Transformer,
+    linkify_html, InlineStyle, SanitizeHtml, Transformer,
 };
 
+const TEXT_PLAIN: &'static str = "text/plain";
+const TEXT_HTML: &'static str = "text/html";
 const IMAGE_JPEG: &'static str = "image/jpeg";
-const IMAGE_PJPEG: &'static str = "image/pjpeg";
 const IMAGE_PNG: &'static str = "image/png";
-const MESSAGE_RFC822: &'static str = "message/rfc822";
 const MULTIPART_ALTERNATIVE: &'static str = "multipart/alternative";
 const MULTIPART_MIXED: &'static str = "multipart/mixed";
 const MULTIPART_RELATED: &'static str = "multipart/related";
-const TEXT_HTML: &'static str = "text/html";
-const TEXT_PLAIN: &'static str = "text/plain";
 
 const MAX_RAW_MESSAGE_SIZE: usize = 100_000;
 
-fn is_notmuch_query(query: &Query) -> bool {
-    query.is_notmuch || query.corpus == Some(Corpus::Notmuch)
-}
-
-pub fn is_notmuch_thread_or_id(id: &str) -> bool {
-    id.starts_with("id:") || id.starts_with("thread:")
-}
-
 // TODO(wathiede): decide good error type
 pub fn threadset_to_messages(thread_set: notmuch::ThreadSet) -> Result<Vec<Message>, ServerError> {
     for t in thread_set.0 {
@@ -50,28 +38,18 @@ pub fn threadset_to_messages(thread_set: notmuch::ThreadSet) -> Result<Vec<Messa
     Ok(Vec::new())
 }
 
-#[instrument(name="nm::count", skip_all, fields(query=%query))]
+pub async fn count(nm: &Notmuch, query: &str) -> Result<usize, ServerError> {
-pub async fn count(nm: &Notmuch, query: &Query) -> Result<usize, ServerError> {
+    Ok(nm.count(query)?)
-    if !is_notmuch_query(query) {
-        return Ok(0);
-    }
-    let query = query.to_notmuch();
-    Ok(nm.count(&query)?)
 }
 
-#[instrument(name="nm::search", skip_all, fields(query=%query))]
 pub async fn search(
     nm: &Notmuch,
     after: Option<i32>,
     before: Option<i32>,
     first: Option<i32>,
     last: Option<i32>,
-    query: &Query,
+    query: String,
 ) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
-    if !is_notmuch_query(query) {
-        return Ok(Vec::new());
-    }
-    let query = query.to_notmuch();
     let (offset, mut limit) = compute_offset_limit(after, before, first, last);
     if before.is_none() {
         // When searching forward, the +1 is to see if there are more pages of data available.
@@ -96,14 +74,12 @@ pub async fn search(
                     authors: ts.authors,
                     subject: ts.subject,
                     tags: ts.tags,
-                    corpus: Corpus::Notmuch,
                 },
             )
         })
         .collect())
 }
 
-#[instrument(name="nm::tags", skip_all, fields(needs_unread=needs_unread))]
 pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     let now = Instant::now();
     let unread_msg_cnt: HashMap<String, usize> = if needs_unread {
@@ -145,10 +121,8 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     Ok(tags)
 }
 
-#[instrument(name="nm::thread", skip_all, fields(thread_id=thread_id))]
 pub async fn thread(
     nm: &Notmuch,
-    pool: &PgPool,
     thread_id: String,
     debug_content_tree: bool,
 ) -> Result<Thread, ServerError> {
@@ -161,7 +135,7 @@ pub async fn thread(
         let mmap = unsafe { MmapOptions::new().map(&file)? };
         let m = parse_mail(&mmap)?;
         let from = email_addresses(&path, &m, "from")?;
-        let mut from = match from.len() {
+        let from = match from.len() {
             0 => None,
             1 => from.into_iter().next(),
             _ => {
@@ -173,16 +147,6 @@ pub async fn thread(
                 from.into_iter().next()
             }
         };
-        match from.as_mut() {
-            Some(from) => {
-                if let Some(addr) = from.addr.as_mut() {
-                    let photo_url = photo_url_for_email_address(&pool, &addr).await?;
-                    from.photo_url = photo_url;
-                }
-            }
-            _ => (),
-        }
-
         let to = email_addresses(&path, &m, "to")?;
         let cc = email_addresses(&path, &m, "cc")?;
         let subject = m.headers.get_first_value("subject");
@@ -192,9 +156,7 @@ pub async fn thread(
             .and_then(|d| mailparse::dateparse(&d).ok());
         let cid_prefix = shared::urls::cid_prefix(None, &id);
         let base_url = None;
-        let mut part_addr = Vec::new();
+        let body = match extract_body(&m, &id)? {
-        part_addr.push(id.to_string());
-        let body = match extract_body(&m, &mut part_addr)? {
             Body::PlainText(PlainText { text, content_tree }) => {
                 let text = if text.len() > MAX_RAW_MESSAGE_SIZE {
                     format!(
@@ -341,7 +303,6 @@ fn email_addresses(
                         mailparse::MailAddr::Single(s) => addrs.push(Email {
                             name: s.display_name,
                             addr: Some(s.addr),
-                            photo_url: None,
                         }), //println!("Single: {s}"),
                     }
                 }
@@ -356,14 +317,12 @@ fn email_addresses(
                         addrs.push(Email {
                             name: Some(name.to_string()),
                             addr: Some(addr.to_string()),
-                            photo_url: None,
                         });
                     }
                 } else {
                     addrs.push(Email {
                         name: Some(v),
                         addr: None,
-                        photo_url: None,
                     });
                 }
             }
@@ -424,14 +383,16 @@ pub fn attachment_bytes(nm: &Notmuch, id: &str, idx: &[usize]) -> Result<Attachm
     Err(ServerError::PartNotFound)
 }
 
-fn extract_body(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+fn extract_body(m: &ParsedMail, id: &str) -> Result<Body, ServerError> {
+    let mut part_addr = Vec::new();
+    part_addr.push(id.to_string());
     let body = m.get_body()?;
     let ret = match m.ctype.mimetype.as_str() {
         TEXT_PLAIN => return Ok(Body::text(body)),
         TEXT_HTML => return Ok(Body::html(body)),
-        MULTIPART_MIXED => extract_mixed(m, part_addr),
+        MULTIPART_MIXED => extract_mixed(m, &mut part_addr),
-        MULTIPART_ALTERNATIVE => extract_alternative(m, part_addr),
+        MULTIPART_ALTERNATIVE => extract_alternative(m, &mut part_addr),
-        MULTIPART_RELATED => extract_related(m, part_addr),
+        MULTIPART_RELATED => extract_related(m, &mut part_addr),
         _ => extract_unhandled(m),
     };
     if let Err(err) = ret {
@@ -471,7 +432,7 @@ fn extract_alternative(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Bo
     }
     for sp in &m.subparts {
         if sp.ctype.mimetype.as_str() == MULTIPART_MIXED {
-            return extract_mixed(sp, part_addr);
+            return extract_related(sp, part_addr);
         }
     }
     for sp in &m.subparts {
@@ -500,16 +461,13 @@ fn extract_alternative(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Bo
 // multipart/mixed defines multiple types of context all of which should be presented to the user
 // 'serially'.
 fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
-    //todo!("add some sort of visual indicator there are unhandled types, i.e. .ics files");
     let handled_types = vec![
-        IMAGE_JPEG,
-        IMAGE_PJPEG,
-        IMAGE_PNG,
-        MESSAGE_RFC822,
         MULTIPART_ALTERNATIVE,
         MULTIPART_RELATED,
         TEXT_HTML,
         TEXT_PLAIN,
+        IMAGE_JPEG,
+        IMAGE_PNG,
     ];
     let mut unhandled_types: Vec<_> = m
         .subparts
@@ -525,12 +483,11 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
     for (idx, sp) in m.subparts.iter().enumerate() {
         part_addr.push(idx.to_string());
         match sp.ctype.mimetype.as_str() {
-            MESSAGE_RFC822 => parts.push(extract_rfc822(&sp, part_addr)?),
             MULTIPART_RELATED => parts.push(extract_related(sp, part_addr)?),
             MULTIPART_ALTERNATIVE => parts.push(extract_alternative(sp, part_addr)?),
             TEXT_PLAIN => parts.push(Body::text(sp.get_body()?)),
             TEXT_HTML => parts.push(Body::html(sp.get_body()?)),
-            IMAGE_PJPEG | IMAGE_JPEG | IMAGE_PNG => {
+            IMAGE_JPEG | IMAGE_PNG => {
                 let pcd = sp.get_content_disposition();
                 let filename = pcd
                     .params
@@ -553,25 +510,13 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
                     )));
                 }
             }
-            mt => parts.push(unhandled_html(MULTIPART_MIXED, mt)),
+            _ => (),
         }
         part_addr.pop();
     }
     Ok(flatten_body_parts(&parts))
 }
 
-fn unhandled_html(parent_type: &str, child_type: &str) -> Body {
-    Body::Html(Html {
-        html: format!(
-            r#"
-<div class="p-4 error">
-Unhandled mimetype {child_type} in a {parent_type} message
-</div>
-    "#
-        ),
-        content_tree: String::new(),
-    })
-}
 fn flatten_body_parts(parts: &[Body]) -> Body {
     let html = parts
         .iter()
@@ -582,7 +527,7 @@ fn flatten_body_parts(parts: &[Body]) -> Body {
                     // Trim newlines to prevent excessive white space at the beginning/end of
                     // presenation. Leave tabs and spaces incase plain text attempts to center a
                     // header on the first line.
-                    linkify_html(&html_escape::encode_text(text).trim_matches('\n'))
+                    linkify_html(&text.trim_matches('\n'))
                 )
             }
             Body::Html(Html { html, .. }) => html.clone(),
@@ -593,14 +538,14 @@ fn flatten_body_parts(parts: &[Body]) -> Body {
                     // Trim newlines to prevent excessive white space at the beginning/end of
                     // presenation. Leave tabs and spaces incase plain text attempts to center a
                     // header on the first line.
-                    linkify_html(&html_escape::encode_text(text).trim_matches('\n'))
+                    linkify_html(&text.trim_matches('\n'))
                 )
             }
         })
         .collect::<Vec<_>>()
         .join("\n");
 
-    info!("flatten_body_parts {}", parts.len());
+    info!("flatten_body_parts {} {html}", parts.len());
     Body::html(html)
 }
 
@@ -611,7 +556,6 @@ fn extract_related(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body,
         TEXT_HTML,
         TEXT_PLAIN,
         IMAGE_JPEG,
-        IMAGE_PJPEG,
         IMAGE_PNG,
     ];
     let mut unhandled_types: Vec<_> = m
@@ -626,10 +570,7 @@ fn extract_related(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body,
     }
 
     for (i, sp) in m.subparts.iter().enumerate() {
-        if sp.ctype.mimetype == IMAGE_PNG
+        if sp.ctype.mimetype == IMAGE_PNG || sp.ctype.mimetype == IMAGE_JPEG {
-            || sp.ctype.mimetype == IMAGE_JPEG
-            || sp.ctype.mimetype == IMAGE_PJPEG
-        {
             info!("sp.ctype {:#?}", sp.ctype);
             //info!("sp.headers {:#?}", sp.headers);
             if let Some(cid) = sp.headers.get_first_value("Content-Id") {
@@ -696,7 +637,6 @@ fn walk_attachments_inner<T, F: Fn(&ParsedMail, &[usize]) -> Option<T> + Copy>(
 fn extract_attachments(m: &ParsedMail, id: &str) -> Result<Vec<Attachment>, ServerError> {
     let mut attachments = Vec::new();
     for (idx, sp) in m.subparts.iter().enumerate() {
-        info!("sp: {:?}", sp.headers);
         if let Some(attachment) = extract_attachment(sp, id, &[idx]) {
             // Filter out inline attachements, they're flattened into the body of the message.
             if attachment.disposition == DispositionType::Attachment {
@@ -709,22 +649,11 @@ fn extract_attachments(m: &ParsedMail, id: &str) -> Result<Vec<Attachment>, Serv
 
 fn extract_attachment(m: &ParsedMail, id: &str, idx: &[usize]) -> Option<Attachment> {
     let pcd = m.get_content_disposition();
-    let pct = m
+    // TODO: do we need to handle empty filename attachments, or should we change the definition of
-        .get_headers()
+    // Attachment::filename?
-        .get_first_value("Content-Type")
+    let Some(filename) = pcd.params.get("filename").map(|f| f.clone()) else {
-        .map(|s| parse_content_type(&s));
+        return None;
-    let filename = match (
-        pcd.params.get("filename").map(|f| f.clone()),
-        pct.map(|pct| pct.params.get("name").map(|f| f.clone())),
-    ) {
-        // Use filename from Content-Disposition
-        (Some(filename), _) => filename,
-        // Use filename from Content-Type
-        (_, Some(Some(name))) => name,
-        // No known filename, assume it's not an attachment
-        _ => return None,
     };
-    info!("filename {filename}");
 
     // TODO: grab this from somewhere
     let content_id = None;
@@ -752,44 +681,6 @@ fn extract_attachment(m: &ParsedMail, id: &str, idx: &[usize]) -> Option<Attachm
         bytes,
     });
 }
-fn email_address_strings(emails: &[Email]) -> Vec<String> {
-    emails
-        .iter()
-        .map(|e| e.to_string())
-        .inspect(|e| info!("e {e}"))
-        .collect()
-}
-
-fn extract_rfc822(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
-    fn extract_headers(m: &ParsedMail) -> Result<Body, ServerError> {
-        let path = "<in-memory>";
-        let from = email_address_strings(&email_addresses(path, &m, "from")?).join(", ");
-        let to = email_address_strings(&email_addresses(path, &m, "to")?).join(", ");
-        let cc = email_address_strings(&email_addresses(path, &m, "cc")?).join(", ");
-        let date = m.headers.get_first_value("date").unwrap_or(String::new());
-        let subject = m
-            .headers
-            .get_first_value("subject")
-            .unwrap_or(String::new());
-        let text = format!(
-            r#"
----------- Forwarded message ----------
-From: {from}
-To: {to}
-CC: {cc}
-Date: {date}
-Subject: {subject}
-"#
-        );
-        Ok(Body::text(text))
-    }
-    let inner_body = m.get_body()?;
-    let inner_m = parse_mail(inner_body.as_bytes())?;
-    let headers = extract_headers(&inner_m)?;
-    let body = extract_body(&inner_m, part_addr)?;
-
-    Ok(flatten_body_parts(&[headers, body]))
-}
 
 pub fn get_attachment_filename(header_value: &str) -> &str {
     info!("get_attachment_filename {header_value}");
@@ -871,45 +762,15 @@ fn render_content_type_tree(m: &ParsedMail) -> String {
     )
 }
 
-#[instrument(name="nm::set_read_status", skip_all, fields(query=%query, unread=unread))]
 pub async fn set_read_status<'ctx>(
     nm: &Notmuch,
-    query: &Query,
+    query: &str,
     unread: bool,
 ) -> Result<bool, ServerError> {
-    let uids: Vec<_> = query
+    if unread {
-        .uids
+        nm.tag_add("unread", &format!("{query}"))?;
-        .iter()
+    } else {
-        .filter(|uid| is_notmuch_thread_or_id(uid))
+        nm.tag_remove("unread", &format!("{query}"))?;
-        .collect();
-    info!("set_read_status({unread} {uids:?})");
-    for uid in uids {
-        if unread {
-            nm.tag_add("unread", uid)?;
-        } else {
-            nm.tag_remove("unread", uid)?;
-        }
     }
     Ok(true)
 }
-
-async fn photo_url_for_email_address(
-    pool: &PgPool,
-    addr: &str,
-) -> Result<Option<String>, ServerError> {
-    let row = sqlx::query!(
-        r#"
-SELECT
-  url
-FROM email_photo ep
-JOIN email_address ea
-ON ep.id = ea.email_photo_id
-WHERE
-  address = $1
-    "#,
-        addr
-    )
-    .fetch_optional(pool)
-    .await?;
-    Ok(row.map(|r| r.url))
-}

server/src/tantivy.rs

@@ -1,353 +0,0 @@
-use std::collections::HashSet;
-
-use log::{debug, error, info, warn};
-use sqlx::{postgres::PgPool, types::time::PrimitiveDateTime};
-use tantivy::{
-    collector::{DocSetCollector, TopDocs},
-    doc, query,
-    query::{AllQuery, BooleanQuery, Occur, QueryParser, TermQuery},
-    schema::{Facet, IndexRecordOption, Value},
-    DocAddress, Index, IndexReader, Searcher, TantivyDocument, TantivyError, Term,
-};
-use tracing::{info_span, instrument, Instrument};
-
-use crate::{
-    compute_offset_limit,
-    error::ServerError,
-    graphql::{Corpus, ThreadSummary},
-    newsreader::{extract_thread_id, is_newsreader_thread},
-    thread_summary_from_row, Query, ThreadSummaryRecord,
-};
-
-pub fn is_tantivy_query(query: &Query) -> bool {
-    query.is_tantivy || query.corpus == Some(Corpus::Tantivy)
-}
-pub struct TantivyConnection {
-    db_path: String,
-    index: Index,
-    reader: IndexReader,
-}
-
-fn get_index(db_path: &str) -> Result<Index, TantivyError> {
-    Ok(match Index::open_in_dir(db_path) {
-        Ok(idx) => idx,
-        Err(err) => {
-            warn!("Failed to open {db_path}: {err}");
-            create_news_db(db_path)?;
-            Index::open_in_dir(db_path)?
-        }
-    })
-}
-
-impl TantivyConnection {
-    pub fn new(tantivy_db_path: &str) -> Result<TantivyConnection, TantivyError> {
-        let index = get_index(tantivy_db_path)?;
-        let reader = index.reader()?;
-
-        Ok(TantivyConnection {
-            db_path: tantivy_db_path.to_string(),
-            index,
-            reader,
-        })
-    }
-    #[instrument(name = "tantivy::refresh", skip_all)]
-    pub async fn refresh(&self, pool: &PgPool) -> Result<(), ServerError> {
-        let start_time = std::time::Instant::now();
-        let p_uids: Vec<_> = sqlx::query_file!("sql/all-uids.sql")
-            .fetch_all(pool)
-            .instrument(info_span!("postgres query"))
-            .await?
-            .into_iter()
-            .map(|r| r.uid)
-            .collect();
-        info!(
-            "refresh from postgres got {} uids in {}",
-            p_uids.len(),
-            start_time.elapsed().as_secs_f32()
-        );
-
-        let t_span = info_span!("tantivy query");
-        let _enter = t_span.enter();
-        let start_time = std::time::Instant::now();
-        let (searcher, _query) = self.searcher_and_query(&Query::default())?;
-        let docs = searcher.search(&AllQuery, &DocSetCollector)?;
-        let uid = self.index.schema().get_field("uid")?;
-        let t_uids: Vec<_> = docs
-            .into_iter()
-            .map(|doc_address| {
-                searcher
-                    .doc(doc_address)
-                    .map(|doc: TantivyDocument| {
-                        debug!("doc: {doc:#?}");
-                        doc.get_first(uid)
-                            .expect("uid")
-                            .as_str()
-                            .expect("as_str")
-                            .to_string()
-                    })
-                    .expect("searcher.doc")
-            })
-            .collect();
-        drop(_enter);
-
-        info!(
-            "refresh tantivy got {} uids in {}",
-            t_uids.len(),
-            start_time.elapsed().as_secs_f32()
-        );
-        let t_set: HashSet<_> = t_uids.into_iter().collect();
-        let need: Vec<_> = p_uids
-            .into_iter()
-            .filter(|uid| !t_set.contains(uid.as_str()))
-            .collect();
-        if !need.is_empty() {
-            info!(
-                "need to reindex {} uids: {:?}...",
-                need.len(),
-                &need[..need.len().min(10)]
-            );
-        }
-        let batch_size = 1000;
-        let uids: Vec<_> = need[..need.len().min(batch_size)]
-            .into_iter()
-            .cloned()
-            .collect();
-        self.reindex_uids(pool, &uids).await
-    }
-    #[instrument(skip(self, pool))]
-    async fn reindex_uids(&self, pool: &PgPool, uids: &[String]) -> Result<(), ServerError> {
-        if uids.is_empty() {
-            return Ok(());
-        }
-        // TODO: add SlurpContents and convert HTML to text
-
-        let pool: &PgPool = pool;
-
-        let mut index_writer = self.index.writer(50_000_000)?;
-        let schema = self.index.schema();
-        let site = schema.get_field("site")?;
-        let title = schema.get_field("title")?;
-        let summary = schema.get_field("summary")?;
-        let link = schema.get_field("link")?;
-        let date = schema.get_field("date")?;
-        let is_read = schema.get_field("is_read")?;
-        let uid = schema.get_field("uid")?;
-        let id = schema.get_field("id")?;
-        let tag = schema.get_field("tag")?;
-
-        info!("reindexing {} posts", uids.len());
-        let rows = sqlx::query_file_as!(PostgresDoc, "sql/posts-from-uids.sql", uids)
-            .fetch_all(pool)
-            .await?;
-
-        if uids.len() != rows.len() {
-            error!(
-                "Had {} uids and only got {} rows: uids {uids:?}",
-                uids.len(),
-                rows.len()
-            );
-        }
-        for r in rows {
-            let id_term = Term::from_field_text(uid, &r.uid);
-            index_writer.delete_term(id_term);
-            let slug = r.site;
-            let tag_facet = Facet::from(&format!("/News/{slug}"));
-            index_writer.add_document(doc!(
-                site => slug.clone(),
-                title => r.title,
-                // TODO: clean and extract text from HTML
-                summary => r.summary,
-                link => r.link,
-                date => tantivy::DateTime::from_primitive(r.date),
-                is_read => r.is_read,
-                uid => r.uid,
-                id => r.id as u64,
-                tag => tag_facet,
-            ))?;
-        }
-
-        info_span!("IndexWriter.commit").in_scope(|| index_writer.commit())?;
-        info_span!("IndexReader.reload").in_scope(|| self.reader.reload())?;
-        Ok(())
-    }
-    #[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=%query))]
-    pub async fn reindex_thread(&self, pool: &PgPool, query: &Query) -> Result<(), ServerError> {
-        let uids: Vec<_> = query
-            .uids
-            .iter()
-            .filter(|uid| is_newsreader_thread(uid))
-            .map(|uid| extract_thread_id(uid).to_string())
-            .collect();
-        Ok(self.reindex_uids(pool, &uids).await?)
-    }
-    #[instrument(name = "tantivy::reindex_all", skip_all)]
-    pub async fn reindex_all(&self, pool: &PgPool) -> Result<(), ServerError> {
-        let rows = sqlx::query_file!("sql/all-posts.sql")
-            .fetch_all(pool)
-            .await?;
-
-        let uids: Vec<String> = rows.into_iter().map(|r| r.uid).collect();
-        self.reindex_uids(pool, &uids).await?;
-        Ok(())
-    }
-    fn searcher_and_query(
-        &self,
-        query: &Query,
-    ) -> Result<(Searcher, Box<dyn query::Query>), ServerError> {
-        // TODO: only create one reader
-        // From https://tantivy-search.github.io/examples/basic_search.html
-        // "For a search server you will typically create one reader for the entire lifetime of
-        // your program, and acquire a new searcher for every single request."
-        //
-        // I think there's some challenge in making the reader work if we reindex, so reader my
-        // need to be stored indirectly, and be recreated on reindex
-        // I think creating a reader takes 200-300 ms.
-        let schema = self.index.schema();
-        let searcher = self.reader.searcher();
-        let title = schema.get_field("title")?;
-        let summary = schema.get_field("summary")?;
-        let query_parser = QueryParser::for_index(&self.index, vec![title, summary]);
-        // Tantivy uses '*' to match all docs, not empty string
-        let term = &query.remainder.join(" ");
-        let term = if term.is_empty() { "*" } else { term };
-        info!("query_parser('{term}')");
-
-        let tantivy_query = query_parser.parse_query(&term)?;
-
-        let tag = schema.get_field("tag")?;
-        let is_read = schema.get_field("is_read")?;
-        let mut terms = vec![(Occur::Must, tantivy_query)];
-        for t in &query.tags {
-            let facet = Facet::from(&format!("/{t}"));
-            let facet_term = Term::from_facet(tag, &facet);
-            let facet_term_query = Box::new(TermQuery::new(facet_term, IndexRecordOption::Basic));
-            terms.push((Occur::Must, facet_term_query));
-        }
-        if query.unread_only {
-            info!("searching for unread only");
-            let term = Term::from_field_bool(is_read, false);
-            terms.push((
-                Occur::Must,
-                Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
-            ));
-        }
-        let search_query = BooleanQuery::new(terms);
-        Ok((searcher, Box::new(search_query)))
-    }
-
-    #[instrument(name="tantivy::count", skip_all, fields(query=%query))]
-    pub async fn count(&self, query: &Query) -> Result<usize, ServerError> {
-        if !is_tantivy_query(query) {
-            return Ok(0);
-        }
-        info!("tantivy::count {query:?}");
-        use tantivy::collector::Count;
-        let (searcher, query) = self.searcher_and_query(&query)?;
-        Ok(searcher.search(&query, &Count)?)
-    }
-    #[instrument(name="tantivy::search", skip_all, fields(query=%query))]
-    pub async fn search(
-        &self,
-        pool: &PgPool,
-        after: Option<i32>,
-        before: Option<i32>,
-        first: Option<i32>,
-        last: Option<i32>,
-        query: &Query,
-    ) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
-        if !is_tantivy_query(query) {
-            return Ok(Vec::new());
-        }
-        let (offset, mut limit) = compute_offset_limit(after, before, first, last);
-        if before.is_none() {
-            // When searching forward, the +1 is to see if there are more pages of data available.
-            // Searching backwards implies there's more pages forward, because the value represented by
-            // `before` is on the next page.
-            limit = limit + 1;
-        }
-
-        let (searcher, search_query) = self.searcher_and_query(&query)?;
-        info!("Tantivy::search(query '{query:?}', off {offset}, lim {limit}, search_query {search_query:?})");
-        let top_docs = searcher.search(
-            &search_query,
-            &TopDocs::with_limit(limit as usize)
-                .and_offset(offset as usize)
-                .order_by_u64_field("date", tantivy::index::Order::Desc),
-        )?;
-        info!("search found {} docs", top_docs.len());
-        let uid = self.index.schema().get_field("uid")?;
-        let uids = top_docs
-            .into_iter()
-            .map(|(_, doc_address): (u64, DocAddress)| {
-                searcher.doc(doc_address).map(|doc: TantivyDocument| {
-                    debug!("doc: {doc:#?}");
-                    doc.get_first(uid)
-                        .expect("doc missing uid")
-                        .as_str()
-                        .expect("doc str missing")
-                        .to_string()
-                })
-            })
-            .collect::<Result<Vec<String>, TantivyError>>()?;
-
-        //let uids = format!("'{}'", uids.join("','"));
-        info!("uids {uids:?}");
-        let rows = sqlx::query_file!("sql/threads-from-uid.sql", &uids as &[String])
-            .fetch_all(pool)
-            .await?;
-        let mut res = Vec::new();
-        info!("found {} hits joining w/ tantivy", rows.len());
-        for (i, r) in rows.into_iter().enumerate() {
-            res.push((
-                i as i32 + offset,
-                thread_summary_from_row(ThreadSummaryRecord {
-                    site: r.site,
-                    date: r.date,
-                    is_read: r.is_read,
-                    title: r.title,
-                    uid: r.uid,
-                    name: r.name,
-                    corpus: Corpus::Tantivy,
-                })
-                .await,
-            ));
-        }
-        Ok(res)
-    }
-    pub fn drop_and_load_index(&self) -> Result<(), TantivyError> {
-        create_news_db(&self.db_path)
-    }
-}
-
-fn create_news_db(tantivy_db_path: &str) -> Result<(), TantivyError> {
-    info!("create_news_db");
-    // Don't care if directory didn't exist
-    let _ = std::fs::remove_dir_all(tantivy_db_path);
-    std::fs::create_dir_all(tantivy_db_path)?;
-    use tantivy::schema::*;
-    let mut schema_builder = Schema::builder();
-    schema_builder.add_text_field("site", STRING | STORED);
-    schema_builder.add_text_field("title", TEXT | STORED);
-    schema_builder.add_text_field("summary", TEXT);
-    schema_builder.add_text_field("link", STRING | STORED);
-    schema_builder.add_date_field("date", FAST | INDEXED | STORED);
-    schema_builder.add_bool_field("is_read", FAST | INDEXED | STORED);
-    schema_builder.add_text_field("uid", STRING | STORED);
-    schema_builder.add_u64_field("id", FAST);
-    schema_builder.add_facet_field("tag", FacetOptions::default());
-
-    let schema = schema_builder.build();
-    Index::create_in_dir(tantivy_db_path, schema)?;
-    Ok(())
-}
-
-struct PostgresDoc {
-    site: String,
-    title: String,
-    summary: String,
-    link: String,
-    date: PrimitiveDateTime,
-    is_read: bool,
-    uid: String,
-    id: i32,
-}

server/static/graphql-playground.html

@@ -1,59 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset=utf-8 />
-  <meta name="viewport" content="user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, minimal-ui">
-  <title>GraphQL Playground</title>
-  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/css/index.css" />
-  <link rel="shortcut icon" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/favicon.png" />
-  <script src="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/js/middleware.js"></script>
-</head>
-
-<body>
-  <div id="root">
-    <style>
-      body {
-        background-color: rgb(23, 42, 58);
-        font-family: Open Sans, sans-serif;
-        height: 90vh;
-      }
-
-      #root {
-        height: 100%;
-        width: 100%;
-        display: flex;
-        align-items: center;
-        justify-content: center;
-      }
-
-      .loading {
-        font-size: 32px;
-        font-weight: 200;
-        color: rgba(255, 255, 255, .6);
-        margin-left: 20px;
-      }
-
-      img {
-        width: 78px;
-        height: 78px;
-      }
-
-      .title {
-        font-weight: 400;
-      }
-    </style>
-    <img src='//cdn.jsdelivr.net/npm/graphql-playground-react/build/logo.png' alt=''>
-    <div class="loading"> Loading
-      <span class="title">GraphQL Playground</span>
-    </div>
-  </div>
-  <script>window.addEventListener('load', function (event) {
-      GraphQLPlayground.init(document.getElementById('root'), {
-        // options as 'endpoint' belong here
-        endpoint: "/api/graphql",
-      })
-    })</script>
-</body>
-
-</html>

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shared"
-version = "0.0.115"
+version = "0.0.29"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

web/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-version = "0.0.115"
+version = "0.0.29"
 name = "letterbox"
 repository = "https://github.com/seed-rs/seed-quickstart"
 authors = ["Bill Thiede <git@xinu.tv>"]
@@ -7,7 +7,7 @@ description = "App Description"
 categories = ["category"]
 license = "MIT"
 readme = "./README.md"
-edition = "2021"
+edition = "2018"
 
 [build-dependencies]
 build-info-build = "0.0.38"
@@ -20,10 +20,10 @@ console_error_panic_hook = "0.1.7"
 log = "0.4.17"
 seed = { version = "0.10.0", features = ["routing"] }
 #seed = "0.9.2"
-console_log = { version = "0.1.0", registry = "xinu" }
+console_log = {git = "http://git-private.h.xinu.tv/wathiede/console_log.git"}
 serde = { version = "1.0.147", features = ["derive"] }
-notmuch = { path = "../notmuch" }
+notmuch = {path = "../notmuch"}
-shared = { path = "../shared" }
+shared = {path = "../shared"}
 itertools = "0.10.5"
 serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
 chrono = "0.4.31"
@@ -33,7 +33,6 @@ seed_hooks = { git = "https://github.com/wathiede/styles_hooks", package = "seed
 gloo-net = { version = "0.4.0", features = ["json", "serde_json"] }
 human_format = "1.1.0"
 build-info = "0.0.38"
-wasm-bindgen = "0.2.95"
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ['-Os']

web/Trunk.toml

@@ -1,5 +1,5 @@
 [build]
-release = false
+release = true
 
 [serve]
 # The address to serve on.
@@ -9,11 +9,6 @@ port = 6758
 [[proxy]]
 backend = "http://localhost:9345/api/"
 
-[[hooks]]
-stage = "pre_build"
-command = "printf"
-command_arguments = ["\\033c"]
-
 #[[hooks]]
 #stage = "pre_build"
 #command = "cargo"

web/build.rs

@@ -1,5 +1,5 @@
 fn main() {
-    // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
+	// Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
-    // and `build_info::format!` in the main program.
+	// and `build_info::format!` in the main program.
-    build_info_build::build_script();
+	build_info_build::build_script();
 }

web/graphql/front_page.graphql

@@ -14,7 +14,6 @@ query FrontPageQuery($query: String!, $after: String $before: String, $first: In
       subject
       authors
       tags
-      corpus
     }
   }
   tags {

web/graphql/refresh.graphql

@@ -1,3 +0,0 @@
-mutation RefreshMutation {
-  refresh
-}

web/graphql/schema.json

@@ -2,28 +2,6 @@
   "data": {
     "__schema": {
       "directives": [
-        {
-          "args": [
-            {
-              "defaultValue": "\"No longer supported\"",
-              "description": "A reason for why it is deprecated, formatted using Markdown syntax",
-              "name": "reason",
-              "type": {
-                "kind": "SCALAR",
-                "name": "String",
-                "ofType": null
-              }
-            }
-          ],
-          "description": "Marks an element of a GraphQL schema as no longer supported.",
-          "locations": [
-            "FIELD_DEFINITION",
-            "ARGUMENT_DEFINITION",
-            "INPUT_FIELD_DEFINITION",
-            "ENUM_VALUE"
-          ],
-          "name": "deprecated"
-        },
         {
           "args": [
             {
@@ -49,14 +27,6 @@
           ],
           "name": "include"
         },
-        {
-          "args": [],
-          "description": "Indicates that an Input Object is a OneOf Input Object (and thus requires\n                        exactly one of its field be provided)",
-          "locations": [
-            "INPUT_OBJECT"
-          ],
-          "name": "oneOf"
-        },
         {
           "args": [
             {
@@ -81,29 +51,6 @@
             "INLINE_FRAGMENT"
           ],
           "name": "skip"
-        },
-        {
-          "args": [
-            {
-              "defaultValue": null,
-              "description": "URL that specifies the behavior of this scalar.",
-              "name": "url",
-              "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "String",
-                  "ofType": null
-                }
-              }
-            }
-          ],
-          "description": "Provides a scalar specification URL for specifying the behavior of custom scalar types.",
-          "locations": [
-            "SCALAR"
-          ],
-          "name": "specifiedBy"
         }
       ],
       "mutationType": {
@@ -285,35 +232,6 @@
           "name": "Boolean",
           "possibleTypes": null
         },
-        {
-          "description": null,
-          "enumValues": [
-            {
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "NOTMUCH"
-            },
-            {
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "NEWSREADER"
-            },
-            {
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "TANTIVY"
-            }
-          ],
-          "fields": null,
-          "inputFields": null,
-          "interfaces": null,
-          "kind": "ENUM",
-          "name": "Corpus",
-          "possibleTypes": null
-        },
         {
           "description": null,
           "enumValues": [
@@ -364,18 +282,6 @@
                 "name": "String",
                 "ofType": null
               }
-            },
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "photoUrl",
-              "type": {
-                "kind": "SCALAR",
-                "name": "String",
-                "ofType": null
-              }
             }
           ],
           "inputFields": null,
@@ -944,22 +850,6 @@
                   "ofType": null
                 }
               }
-            },
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "refresh",
-              "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "Boolean",
-                  "ofType": null
-                }
-              }
             }
           ],
           "inputFields": null,
@@ -1646,22 +1536,6 @@
                   }
                 }
               }
-            },
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "corpus",
-              "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "ENUM",
-                  "name": "Corpus",
-                  "ofType": null
-                }
-              }
             }
           ],
           "inputFields": null,
@@ -1888,22 +1762,7 @@
               }
             },
             {
-              "args": [
+              "args": [],
-                {
-                  "defaultValue": "false",
-                  "description": null,
-                  "name": "includeDeprecated",
-                  "type": {
-                    "kind": "NON_NULL",
-                    "name": null,
-                    "ofType": {
-                      "kind": "SCALAR",
-                      "name": "Boolean",
-                      "ofType": null
-                    }
-                  }
-                }
-              ],
               "deprecationReason": null,
               "description": null,
               "isDeprecated": false,
@@ -2174,22 +2033,7 @@
               }
             },
             {
-              "args": [
+              "args": [],
-                {
-                  "defaultValue": "false",
-                  "description": null,
-                  "name": "includeDeprecated",
-                  "type": {
-                    "kind": "NON_NULL",
-                    "name": null,
-                    "ofType": {
-                      "kind": "SCALAR",
-                      "name": "Boolean",
-                      "ofType": null
-                    }
-                  }
-                }
-              ],
               "deprecationReason": null,
               "description": null,
               "isDeprecated": false,
@@ -2322,34 +2166,6 @@
                 "name": "String",
                 "ofType": null
               }
-            },
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "isDeprecated",
-              "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "Boolean",
-                  "ofType": null
-                }
-              }
-            },
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": null,
-              "isDeprecated": false,
-              "name": "deprecationReason",
-              "type": {
-                "kind": "SCALAR",
-                "name": "String",
-                "ofType": null
-              }
             }
           ],
           "inputFields": null,
@@ -2362,22 +2178,6 @@
           "description": "A GraphQL Schema defines the capabilities of a GraphQL server. It exposes\nall available types and directives on the server, as well as the entry\npoints for query, mutation, and subscription operations.",
           "enumValues": null,
           "fields": [
-            {
-              "args": [],
-              "deprecationReason": null,
-              "description": "description of __Schema for newer graphiql introspection schema\nrequirements",
-              "isDeprecated": false,
-              "name": "description",
-              "type": {
-                "kind": "NON_NULL",
-                "name": null,
-                "ofType": {
-                  "kind": "SCALAR",
-                  "name": "String",
-                  "ofType": null
-                }
-              }
-            },
             {
               "args": [],
               "deprecationReason": null,
@@ -2628,22 +2428,7 @@
               }
             },
             {
-              "args": [
+              "args": [],
-                {
-                  "defaultValue": "false",
-                  "description": null,
-                  "name": "includeDeprecated",
-                  "type": {
-                    "kind": "NON_NULL",
-                    "name": null,
-                    "ofType": {
-                      "kind": "SCALAR",
-                      "name": "Boolean",
-                      "ofType": null
-                    }
-                  }
-                }
-              ],
               "deprecationReason": null,
               "description": null,
               "isDeprecated": false,

web/graphql/show_thread.graphql

@@ -21,7 +21,6 @@ query ShowThreadQuery($threadId: String!) {
         from {
           name
           addr
-          photoUrl
         }
         to {
           name

web/index.html

@@ -4,20 +4,29 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <link rel="stylesheet" href="https://jenil.github.io/bulmaswatch/cyborg/bulmaswatch.min.css">
+  <!-- 
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
+-->
+  <!--
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@1.0.0/css/bulma.min.css">
+-->
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.3.0/css/all.min.css"
     integrity="sha512-SzlrxWUlpfuzQ+pcUCosxcglQRNAq/DZjVsC0lE40xsADsfeQoEypE+enwcOiGjk/bSuGGKHEyjSoQ1zVisanQ=="
     crossorigin="anonymous" referrerpolicy="no-referrer" />
   <link rel="icon" href="https://static.xinu.tv/favicon/letterbox.svg" />
+  <link data-trunk rel="css" href="static/style.css" />
+  <!-- Pretty checkboxes from https://justboil.github.io/bulma-checkbox/ -->
+  <link data-trunk rel="css" href="static/main.css" />
   <!-- tall thin font for user icon -->
   <link rel="preconnect" href="https://fonts.googleapis.com">
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@700&display=swap" rel="stylesheet">
-  <!-- <link data-trunk rel="css" href="static/site-specific.css" /> -->
+  <link data-trunk rel="css" href="static/site-specific.css" />
-  <link data-trunk rel="tailwind-css" href="./src/tailwind.css" />
 </head>
 
 <body>
   <section id="app"></section>
 </body>
 
-</html>
+</html>

web/src/api.rs

@@ -1,3 +1,10 @@
+use gloo_net::{http::Request, Error};
+use log::info;
+
+const BASE_URL: &str = "/api";
+pub fn refresh() -> String {
+    format!("{BASE_URL}/refresh")
+}
 pub mod urls {
     use seed::Url;
     pub fn search(query: &str, page: usize) -> Url {
@@ -12,3 +19,9 @@ pub mod urls {
         Url::new().set_hash_path(["t", tid])
     }
 }
+
+pub async fn refresh_request() -> Result<(), Error> {
+    let t = Request::get(&refresh()).send().await?.text().await?;
+    info!("refresh {t}");
+    Ok(())
+}

web/src/graphql.rs

@@ -44,14 +44,6 @@ pub struct AddTagMutation;
 )]
 pub struct RemoveTagMutation;
 
-#[derive(GraphQLQuery)]
-#[graphql(
-    schema_path = "graphql/schema.json",
-    query_path = "graphql/refresh.graphql",
-    response_derives = "Debug"
-)]
-pub struct RefreshMutation;
-
 pub async fn send_graphql<Body, Resp>(body: Body) -> Result<graphql_client::Response<Resp>, Error>
 where
     Body: Serialize,

web/src/state.rs

@@ -7,6 +7,7 @@ use thiserror::Error;
 use web_sys::HtmlElement;
 
 use crate::{
+    api,
     api::urls,
     consts::SEARCH_RESULTS_PER_PAGE,
     graphql,
@@ -110,17 +111,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
         Msg::Noop => {}
         Msg::RefreshStart => {
             model.refreshing_state = RefreshingState::Loading;
-            orders.perform_cmd(async move {
+            orders.perform_cmd(async move { Msg::RefreshDone(api::refresh_request().await.err()) });
-                Msg::RefreshDone(
-                    send_graphql::<_, graphql::refresh_mutation::ResponseData>(
-                        graphql::RefreshMutation::build_query(
-                            graphql::refresh_mutation::Variables {},
-                        ),
-                    )
-                    .await
-                    .err(),
-                )
-            });
         }
         Msg::RefreshDone(err) => {
             model.refreshing_state = if let Some(err) = err {
@@ -180,12 +171,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 Context::None => (),                // do nothing (yet?)
             };
         }
-        Msg::GoToSearchResults => {
-            let url = urls::search(&model.query, 0);
-            info!("GoToSearchRestuls Start");
-            orders.request_url(url);
-            info!("GoToSearchRestuls End");
-        }
 
         Msg::UpdateQuery(query) => model.query = query,
         Msg::SearchQuery(query) => {
@@ -193,6 +178,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
         }
 
         Msg::SetUnread(query, unread) => {
+            let search_url = urls::search(&model.query, 0).to_string();
             orders.skip().perform_cmd(async move {
                 let res: Result<
                     graphql_client::Response<graphql::mark_read_mutation::ResponseData>,
@@ -207,10 +193,15 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 if let Err(e) = res {
                     error!("Failed to set read for {query} to {unread}: {e}");
                 }
-                Msg::Refresh
+                seed::window()
+                    .location()
+                    .set_href(&search_url)
+                    .expect("failed to change location");
+                Msg::Noop
             });
         }
         Msg::AddTag(query, tag) => {
+            let search_url = urls::search(&model.query, 0).to_string();
             orders.skip().perform_cmd(async move {
                 let res: Result<
                     graphql_client::Response<graphql::add_tag_mutation::ResponseData>,
@@ -225,10 +216,15 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 if let Err(e) = res {
                     error!("Failed to add tag {tag} to {query}: {e}");
                 }
-                Msg::GoToSearchResults
+                seed::window()
+                    .location()
+                    .set_href(&search_url)
+                    .expect("failed to change location");
+                Msg::Noop
             });
         }
         Msg::RemoveTag(query, tag) => {
+            let search_url = urls::search(&model.query, 0).to_string();
             orders.skip().perform_cmd(async move {
                 let res: Result<
                     graphql_client::Response<graphql::remove_tag_mutation::ResponseData>,
@@ -244,7 +240,11 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     error!("Failed to remove tag {tag} to {query}: {e}");
                 }
                 // TODO: reconsider this behavior
-                Msg::GoToSearchResults
+                seed::window()
+                    .location()
+                    .set_href(&search_url)
+                    .expect("failed to change location");
+                Msg::Noop
             });
         }
 
@@ -507,7 +507,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             });
         }
         Msg::WindowScrolled => {
-            info!("WindowScrolled");
             if let Some(el) = model.content_el.get() {
                 let ih = window()
                     .inner_height()
@@ -516,7 +515,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .value_of();
 
                 let r = el.get_bounding_client_rect();
-                info!("r {r:?} ih {ih}");
                 if r.height() < ih {
                     // The whole content fits in the window, no scrollbar
                     orders.send_msg(Msg::SetProgress(0.));
@@ -630,7 +628,6 @@ pub enum Msg {
     RefreshDone(Option<gloo_net::Error>),
     NextPage,
     PreviousPage,
-    GoToSearchResults,
     UpdateQuery(String),
     SearchQuery(String),
 

web/src/tailwind.css

@@ -1,3 +0,0 @@
-@tailwind base;
-@tailwind components;
-@tailwind utilities;

web/src/view/desktop.rs

@@ -0,0 +1,49 @@
+use seed::{prelude::*, *};
+use seed_hooks::topo;
+
+use crate::{
+    graphql::show_thread_query::*,
+    state::{Context, Model, Msg},
+    view::{self, reading_progress, view_header, view_search_results},
+};
+
+#[topo::nested]
+pub(super) fn view(model: &Model) -> Node<Msg> {
+    let show_icon_text = true;
+    // Do two queries, one without `unread` so it loads fast, then a second with unread.
+    let content = match &model.context {
+        Context::None => div![h1!["Loading"]],
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::EmailThread(thread),
+            open_messages,
+        } => view::thread(thread, open_messages, show_icon_text, &model.content_el),
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::NewsPost(post),
+            ..
+        } => view::news_post(post, show_icon_text, &model.content_el),
+        Context::SearchResult {
+            query,
+            results,
+            count,
+            pager,
+            selected_threads,
+        } => view_search_results(
+            &query,
+            results.as_slice(),
+            *count,
+            pager,
+            selected_threads,
+            show_icon_text,
+        ),
+    };
+    div![
+        C!["main-content"],
+        reading_progress(model.read_completion_ratio),
+        div![view::tags(model), view::versions(&model.versions)],
+        div![
+            view_header(&model.query, &model.refreshing_state),
+            content,
+            view_header(&model.query, &model.refreshing_state),
+        ]
+    ]
+}

web/src/view/mobile.rs

@@ -0,0 +1,121 @@
+use std::collections::HashSet;
+
+use seed::{prelude::*, *};
+
+use crate::{
+    api::urls,
+    graphql::{front_page_query::*, show_thread_query::*},
+    state::{Context, Model, Msg},
+    view::{
+        self, human_age, pretty_authors, reading_progress, search_toolbar, set_title, tags_chiclet,
+        view_header,
+    },
+};
+
+pub(super) fn view(model: &Model) -> Node<Msg> {
+    let show_icon_text = false;
+    let content = match &model.context {
+        Context::None => div![h1!["Loading"]],
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::EmailThread(thread),
+            open_messages,
+        } => view::thread(thread, open_messages, show_icon_text, &model.content_el),
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::NewsPost(post),
+            ..
+        } => view::news_post(post, show_icon_text, &model.content_el),
+        Context::SearchResult {
+            query,
+            results,
+            count,
+            pager,
+            selected_threads,
+        } => search_results(
+            &query,
+            results.as_slice(),
+            *count,
+            pager,
+            selected_threads,
+            show_icon_text,
+        ),
+    };
+    div![
+        reading_progress(model.read_completion_ratio),
+        view_header(&model.query, &model.refreshing_state),
+        content,
+        view_header(&model.query, &model.refreshing_state),
+        div![view::tags(model), view::versions(&model.versions)]
+    ]
+}
+
+fn search_results(
+    query: &str,
+    results: &[FrontPageQuerySearchNodes],
+    count: usize,
+    pager: &FrontPageQuerySearchPageInfo,
+    selected_threads: &HashSet<String>,
+    show_icon_text: bool,
+) -> Node<Msg> {
+    if query.is_empty() {
+        set_title("all mail");
+    } else {
+        set_title(query);
+    }
+    let rows = results.iter().map(|r| {
+        let tid = r.thread.clone();
+        let check_tid = r.thread.clone();
+        let datetime = human_age(r.timestamp as i64);
+        let unread_idx = r.tags.iter().position(|e| e == &"unread");
+        let mut tags = r.tags.clone();
+        if let Some(idx) = unread_idx {
+            tags.remove(idx);
+        };
+        div![
+            C!["row"],
+            label![
+                C!["b-checkbox", "checkbox", "is-large"],
+                input![attrs! {
+                    At::Type=>"checkbox",
+                    At::Checked=>selected_threads.contains(&tid).as_at_value(),
+                }],
+                span![C!["check"]],
+                ev(Ev::Input, move |e| {
+                    if let Some(input) = e
+                        .target()
+                        .as_ref()
+                        .expect("failed to get reference to target")
+                        .dyn_ref::<web_sys::HtmlInputElement>()
+                    {
+                        if input.checked() {
+                            Msg::SelectionAddThread(check_tid)
+                        } else {
+                            Msg::SelectionRemoveThread(check_tid)
+                        }
+                    } else {
+                        Msg::Noop
+                    }
+                }),
+            ],
+            a![
+                C!["has-text-light", "summary"],
+                IF!(unread_idx.is_some() => C!["unread"]),
+                attrs! {
+                    At::Href => urls::thread(&tid)
+                },
+                div![C!["subject"], &r.subject],
+                span![C!["from", "is-size-7"], pretty_authors(&r.authors)],
+                div![
+                    span![C!["is-size-7"], tags_chiclet(&tags, true)],
+                    span![C!["is-size-7", "float-right", "date"], datetime]
+                ]
+            ]
+        ]
+    });
+    let show_bulk_edit = !selected_threads.is_empty();
+    div![
+        C!["search-results"],
+        search_toolbar(count, pager, show_bulk_edit, show_icon_text),
+        div![C!["index"], rows],
+        search_toolbar(count, pager, show_bulk_edit, show_icon_text),
+    ]
+}

web/src/view/tablet.rs

@@ -0,0 +1,48 @@
+use seed::{prelude::*, *};
+
+use crate::{
+    graphql::show_thread_query::*,
+    state::{Context, Model, Msg},
+    view::{self, reading_progress, view_header, view_search_results},
+};
+
+pub(super) fn view(model: &Model) -> Node<Msg> {
+    let show_icon_text = false;
+    // Do two queries, one without `unread` so it loads fast, then a second with unread.
+    let content = match &model.context {
+        Context::None => div![h1!["Loading"]],
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::EmailThread(thread),
+            open_messages,
+        } => view::thread(thread, open_messages, show_icon_text, &model.content_el),
+        Context::ThreadResult {
+            thread: ShowThreadQueryThread::NewsPost(post),
+            ..
+        } => view::news_post(post, show_icon_text, &model.content_el),
+        Context::SearchResult {
+            query,
+            results,
+            count,
+            pager,
+            selected_threads,
+        } => view_search_results(
+            &query,
+            results.as_slice(),
+            *count,
+            pager,
+            selected_threads,
+            show_icon_text,
+        ),
+    };
+    div![
+        C!["main-content"],
+        div![
+            reading_progress(model.read_completion_ratio),
+            view_header(&model.query, &model.refreshing_state),
+            content,
+            view_header(&model.query, &model.refreshing_state),
+            view::tags(model),
+            view::versions(&model.versions)
+        ]
+    ]
+}

web/static/main.css

@@ -0,0 +1,268 @@
+/* Bulma Utilities */
+.b-checkbox.checkbox {
+  -webkit-touch-callout: none;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+}
+
+/* Box-shadow on hover */
+.b-checkbox.checkbox {
+  outline: none;
+  display: inline-flex;
+  align-items: center;
+}
+
+.b-checkbox.checkbox:not(.button) {
+  margin-right: 0.5em;
+}
+
+.b-checkbox.checkbox:not(.button) + .checkbox:last-child {
+  margin-right: 0;
+}
+
+.b-checkbox.checkbox input[type=checkbox] {
+  position: absolute;
+  left: 0;
+  opacity: 0;
+  outline: none;
+  z-index: -1;
+}
+
+.b-checkbox.checkbox input[type=checkbox] + .check {
+  width: 1.25em;
+  height: 1.25em;
+  flex-shrink: 0;
+  border-radius: 4px;
+  border: 2px solid #7a7a7a;
+  transition: background 150ms ease-out;
+  background: transparent;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check {
+  background: #00d1b2 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-white {
+  background: white url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%230a0a0a' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: white;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-black {
+  background: #0a0a0a url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:white' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #0a0a0a;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-light {
+  background: whitesmoke url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:rgba(0, 0, 0, 0.7)' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: whitesmoke;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-dark {
+  background: #363636 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #363636;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-primary {
+  background: #00d1b2 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-link {
+  background: #485fc7 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #485fc7;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-info {
+  background: #3e8ed0 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #3e8ed0;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-success {
+  background: #48c78e url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #48c78e;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-warning {
+  background: #ffe08a url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:rgba(0, 0, 0, 0.7)' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #ffe08a;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:checked + .check.is-danger {
+  background: #f14668 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Cpath style='fill:%23fff' d='M 0.04038059,0.6267767 0.14644661,0.52071068 0.42928932,0.80355339 0.3232233,0.90961941 z M 0.21715729,0.80355339 0.85355339,0.16715729 0.95961941,0.2732233 0.3232233,0.90961941 z'%3E%3C/path%3E%3C/svg%3E") no-repeat center center;
+  border-color: #f14668;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check {
+  background: #00d1b2 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-white, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-white {
+  background: white url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%230a0a0a' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: white;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-black, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-black {
+  background: #0a0a0a url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:white' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #0a0a0a;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-light, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-light {
+  background: whitesmoke url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:rgba(0, 0, 0, 0.7)' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: whitesmoke;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-dark, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-dark {
+  background: #363636 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #363636;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-primary, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-primary {
+  background: #00d1b2 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-link, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-link {
+  background: #485fc7 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #485fc7;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-info, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-info {
+  background: #3e8ed0 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #3e8ed0;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-success, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-success {
+  background: #48c78e url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #48c78e;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-warning, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-warning {
+  background: #ffe08a url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:rgba(0, 0, 0, 0.7)' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #ffe08a;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:indeterminate + .check.is-danger, .b-checkbox.checkbox input[type=checkbox].is-indeterminate + .check.is-danger {
+  background: #f14668 url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1 1'%3E%3Crect style='fill:%23fff' width='0.7' height='0.2' x='.15' y='.4'%3E%3C/rect%3E%3C/svg%3E") no-repeat center center;
+  border-color: #f14668;
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus + .check {
+  box-shadow: 0 0 0.5em rgba(122, 122, 122, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check {
+  box-shadow: 0 0 0.5em rgba(0, 209, 178, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-white {
+  box-shadow: 0 0 0.5em rgba(255, 255, 255, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-black {
+  box-shadow: 0 0 0.5em rgba(10, 10, 10, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-light {
+  box-shadow: 0 0 0.5em rgba(245, 245, 245, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-dark {
+  box-shadow: 0 0 0.5em rgba(54, 54, 54, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-primary {
+  box-shadow: 0 0 0.5em rgba(0, 209, 178, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-link {
+  box-shadow: 0 0 0.5em rgba(72, 95, 199, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-info {
+  box-shadow: 0 0 0.5em rgba(62, 142, 208, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-success {
+  box-shadow: 0 0 0.5em rgba(72, 199, 142, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-warning {
+  box-shadow: 0 0 0.5em rgba(255, 224, 138, 0.8);
+}
+
+.b-checkbox.checkbox input[type=checkbox]:focus:checked + .check.is-danger {
+  box-shadow: 0 0 0.5em rgba(241, 70, 104, 0.8);
+}
+
+.b-checkbox.checkbox .control-label {
+  padding-left: calc(0.75em - 1px);
+}
+
+.b-checkbox.checkbox.button {
+  display: flex;
+}
+
+.b-checkbox.checkbox[disabled] {
+  opacity: 0.5;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check {
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-white {
+  border-color: white;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-black {
+  border-color: #0a0a0a;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-light {
+  border-color: whitesmoke;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-dark {
+  border-color: #363636;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-primary {
+  border-color: #00d1b2;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-link {
+  border-color: #485fc7;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-info {
+  border-color: #3e8ed0;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-success {
+  border-color: #48c78e;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-warning {
+  border-color: #ffe08a;
+}
+
+.b-checkbox.checkbox:hover input[type=checkbox]:not(:disabled) + .check.is-danger {
+  border-color: #f14668;
+}
+
+.b-checkbox.checkbox.is-small {
+  border-radius: 2px;
+  font-size: 0.75rem;
+}
+
+.b-checkbox.checkbox.is-medium {
+  font-size: 1.25rem;
+}
+
+.b-checkbox.checkbox.is-large {
+  font-size: 1.5rem;
+}

web/static/site-specific.css

@@ -9,11 +9,6 @@
   padding: inherit !important;
 }
 
-.body.news-post hr {
-  background-color: #aaa !important;
-  margin: .25rem 0 !important;
-}
-
 .body.news-post .number {
   align-items: inherit;
   background-color: inherit;

web/static/style.css

@@ -41,6 +41,38 @@
   }
 }
 
+.message {
+  display: inline-block;
+  padding: 0.5em;
+  width: 100%;
+}
+
+.message .header table td {
+  border: 0;
+  padding: 0;
+}
+
+.message .header .media-right {
+  padding: 1rem;
+}
+
+.message .headers {
+  position: relative;
+  width: 100%;
+}
+
+.message .headers .read-status {
+  position: absolute;
+  right: 1em;
+  top: 0em;
+}
+
+.message .headers .header {
+  overflow: clip;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
 .message .body {
   background: white;
   color: black;

web/tailwind.config.js

@@ -1,9 +0,0 @@
-/** @type {import('tailwindcss').Config} */
-module.exports = {
-  content: ['./src/**/*.rs'],
-  theme: {
-    extend: {},
-  },
-  plugins: [],
-}
-