chore: Release

Show original/delivered To if no xinu.tv addresses in To/CC fields

.cargo/config.toml

@@ -0,0 +1,9 @@
+
+[build]
+rustflags = [ "--cfg=web_sys_unstable_apis" ]
+
+[registry]
+global-credential-providers = ["cargo:token"]
+
+[registries.xinu]
+index = "sparse+https://git.z.xinu.tv/api/packages/wathiede/cargo/"

.envrc

@@ -0,0 +1,10 @@
+source_up 
+
+export DATABASE_USER="newsreader";
+export DATABASE_NAME="newsreader";
+export DATABASE_HOST="nixos-07.h.xinu.tv";
+export DATABASE_URL="postgres://${DATABASE_USER}@${DATABASE_HOST}/${DATABASE_NAME}";
+export PROD_DATABASE_USER="newsreader";
+export PROD_DATABASE_NAME="newsreader";
+export PROD_DATABASE_HOST="postgres.h.xinu.tv";
+export PROD_DATABASE_URL="postgres://${PROD_DATABASE_USER}@${PROD_DATABASE_HOST}/${PROD_DATABASE_NAME}";

.gitea/workflows/rust.yml

@@ -0,0 +1,67 @@
+on: [push, pull_request]
+
+name: Continuous integration
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo test
+
+  trunk:
+    name: Trunk
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+            toolchain: stable
+            target: wasm32-unknown-unknown
+      - run: cargo install trunk
+      - run: cd web; trunk build
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+      - name: Rustfmt Check
+        uses: actions-rust-lang/rustfmt@v1
+
+  build:
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build
+
+  udeps:
+    name: Disallow unused dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+
+      - name: Run cargo-udeps
+        uses: aig787/cargo-udeps-action@v1
+        with:
+          version: 'latest'
+          args: '--all-targets'
+

Cargo.toml

@@ -1,12 +1,18 @@
 [workspace]
 resolver = "2"
-members = [
-  "web",
-  "server",
-  "notmuch",
-  "procmail2notmuch",
-  "shared"
-]
+default-members = ["server"]
+members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+
+[workspace.package]
+authors = ["Bill Thiede <git@xinu.tv>"]
+edition = "2021"
+license = "UNLICENSED"
+publish = ["xinu"]
+version = "0.9.3"
+repository = "https://git.z.xinu.tv/wathiede/letterbox"
+
+[profile.dev]
+opt-level = 1
 
 [profile.release]
 lto = true

Justfile

@@ -0,0 +1,16 @@
+default:
+    @echo "Run: just patch|minor|major"
+
+major: (_release "major")
+minor: (_release "minor")
+patch: (_release "patch")
+
+sqlx-prepare:
+	cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true
+
+pull:
+    git pull
+
+
+_release level: pull sqlx-prepare
+    cargo-release release  -x {{ level }} --workspace --no-confirm --registry=xinu

Makefile

@@ -0,0 +1,7 @@
+.PHONEY: release
+release:
+	(cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true)
+	bash scripts/update-crate-version.sh
+	git push
+
+all: release

dev.sh

@@ -1,7 +1,7 @@
 cd -- "$( dirname -- "${BASH_SOURCE[0]}" )"
 tmux new-session -d -s letterbox-dev
 tmux rename-window web
-tmux send-keys "cd web; trunk serve -w ../shared -w ../notmuch -w ./" C-m
+tmux send-keys "cd web; trunk serve -w ../.git -w ../shared -w ../notmuch -w ./" C-m
 tmux new-window -n server
-tmux send-keys "cd server; cargo watch -c -x run -w ../shared -w ../notmuch -w ./" C-m
+tmux send-keys "cd server; cargo watch -c -x run -w ../.git -w ../shared -w ../notmuch -w ./" C-m
 tmux attach -d -t letterbox-dev

notmuch/Cargo.toml

@@ -1,17 +1,24 @@
 [package]
-name = "notmuch"
-version = "0.1.0"
-edition = "2021"
+name = "letterbox-notmuch"
+exclude = ["/testdata"]
+description = "Wrapper for calling notmuch cli"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 log = "0.4.14"
+mailparse = "0.16.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
-thiserror = "1.0.30"
+thiserror = "2.0.0"
+tracing = "0.1.41"
 
 [dev-dependencies]
-itertools = "0.10.1"
+itertools = "0.14.0"
 pretty_assertions = "1"
 rayon = "1.5"

notmuch/src/lib.rs

@@ -207,14 +207,16 @@
 //! ```
 
 use std::{
+    collections::HashMap,
     ffi::OsStr,
     io::{self},
     path::{Path, PathBuf},
     process::Command,
 };
 
-use log::info;
+use log::{error, info};
 use serde::{Deserialize, Serialize};
+use tracing::instrument;
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -269,6 +271,12 @@ pub struct Headers {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub bcc: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "Delivered-To")]
+    pub delivered_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "X-Original-To")]
+    pub x_original_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub reply_to: Option<String>,
     pub date: String,
 }
@@ -458,6 +466,8 @@ pub enum NotmuchError {
     StringUtf8Error(#[from] std::string::FromUtf8Error),
     #[error("failed to parse str as int")]
     ParseIntError(#[from] std::num::ParseIntError),
+    #[error("failed to parse mail: {0}")]
+    MailParseError(#[from] mailparse::MailParseError),
 }
 
 #[derive(Default)]
@@ -465,6 +475,8 @@ pub struct Notmuch {
     config_path: Option<PathBuf>,
 }
 
+// TODO: rewrite to use tokio::process::Command and make everything async to see if that helps with
+// concurrency being more parallel.
 impl Notmuch {
     pub fn with_config<P: AsRef<Path>>(config_path: P) -> Notmuch {
         Notmuch {
@@ -472,6 +484,7 @@ impl Notmuch {
         }
     }
 
+    #[instrument(skip_all)]
     pub fn new(&self) -> Result<Vec<u8>, NotmuchError> {
         self.run_notmuch(["new"])
     }
@@ -480,11 +493,29 @@ impl Notmuch {
         self.run_notmuch(std::iter::empty::<&str>())
     }
 
-    pub fn tags(&self) -> Result<Vec<String>, NotmuchError> {
-        let res = self.run_notmuch(["search", "--format=json", "--output=tags", "*"])?;
+    #[instrument(skip_all, fields(query=query))]
+    pub fn tags_for_query(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
+        let res = self.run_notmuch(["search", "--format=json", "--output=tags", query])?;
         Ok(serde_json::from_slice(&res)?)
     }
 
+    pub fn tags(&self) -> Result<Vec<String>, NotmuchError> {
+        self.tags_for_query("*")
+    }
+
+    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
+    pub fn tag_add(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
+        self.run_notmuch(["tag", &format!("+{tag}"), search_term])?;
+        Ok(())
+    }
+
+    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
+    pub fn tag_remove(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
+        self.run_notmuch(["tag", &format!("-{tag}"), search_term])?;
+        Ok(())
+    }
+
+    #[instrument(skip_all, fields(query=query,offset=offset,limit=limit))]
     pub fn search(
         &self,
         query: &str,
@@ -493,28 +524,40 @@ impl Notmuch {
     ) -> Result<SearchSummary, NotmuchError> {
         let query = if query.is_empty() { "*" } else { query };
 
-        let res = self.run_notmuch([
-            "search",
-            "--format=json",
-            &format!("--offset={offset}"),
-            &format!("--limit={limit}"),
-            query,
-        ])?;
-        Ok(serde_json::from_slice(&res)?)
+        let res = self
+            .run_notmuch([
+                "search",
+                "--format=json",
+                &format!("--offset={offset}"),
+                &format!("--limit={limit}"),
+                query,
+            ])
+            .inspect_err(|err| error!("failed to notmuch search for query '{query}': {err}"))?;
+        Ok(serde_json::from_slice(&res).unwrap_or_else(|err| {
+            error!("failed to decode search result for query '{query}': {err}");
+            SearchSummary(Vec::new())
+        }))
     }
 
+    #[instrument(skip_all, fields(query=query))]
     pub fn count(&self, query: &str) -> Result<usize, NotmuchError> {
+        // NOTE: --output=threads is technically more correct, but really slow
+        // TODO: find a fast thread count path
+        // let res = self.run_notmuch(["count", "--output=threads", query])?;
         let res = self.run_notmuch(["count", query])?;
         // Strip '\n' from res.
-        let s = std::str::from_utf8(&res[..res.len() - 1])?;
-        Ok(s.parse()?)
+        let s = std::str::from_utf8(&res)?.trim();
+        Ok(s.parse()
+            .inspect_err(|err| error!("failed to parse count for query '{query}': {err}"))
+            .unwrap_or(0))
     }
 
+    #[instrument(skip_all, fields(query=query))]
     pub fn show(&self, query: &str) -> Result<ThreadSet, NotmuchError> {
         let slice = self.run_notmuch([
             "show",
             "--include-html=true",
-            "--entire-thread=true",
+            "--entire-thread=false",
             "--format=json",
             query,
         ])?;
@@ -528,6 +571,7 @@ impl Notmuch {
         Ok(val)
     }
 
+    #[instrument(skip_all, fields(query=query,part=part))]
     pub fn show_part(&self, query: &str, part: usize) -> Result<Part, NotmuchError> {
         let slice = self.run_notmuch([
             "show",
@@ -547,25 +591,103 @@ impl Notmuch {
         Ok(val)
     }
 
+    #[instrument(skip_all, fields(id=id))]
     pub fn show_original(&self, id: &MessageId) -> Result<Vec<u8>, NotmuchError> {
         self.show_original_part(id, 0)
     }
 
+    #[instrument(skip_all, fields(id=id,part=part))]
     pub fn show_original_part(&self, id: &MessageId, part: usize) -> Result<Vec<u8>, NotmuchError> {
         let res = self.run_notmuch(["show", "--part", &part.to_string(), id])?;
         Ok(res)
     }
 
+    #[instrument(skip_all, fields(query=query))]
     pub fn message_ids(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
         let res = self.run_notmuch(["search", "--output=messages", "--format=json", query])?;
         Ok(serde_json::from_slice(&res)?)
     }
 
+    #[instrument(skip_all, fields(query=query))]
     pub fn files(&self, query: &str) -> Result<Vec<String>, NotmuchError> {
         let res = self.run_notmuch(["search", "--output=files", "--format=json", query])?;
         Ok(serde_json::from_slice(&res)?)
     }
 
+    #[instrument(skip_all)]
+    pub fn unread_recipients(&self) -> Result<HashMap<String, usize>, NotmuchError> {
+        let slice = self.run_notmuch([
+            "show",
+            "--include-html=false",
+            "--entire-thread=false",
+            "--body=false",
+            "--format=json",
+            // Arbitrary limit to prevent too much work
+            "--limit=1000",
+            "is:unread",
+        ])?;
+        // Notmuch returns JSON with invalid unicode.  So we lossy convert it to a string here and
+        // use that for parsing in rust.
+        let s = String::from_utf8_lossy(&slice);
+        let mut deserializer = serde_json::Deserializer::from_str(&s);
+        deserializer.disable_recursion_limit();
+        let ts: ThreadSet = serde::de::Deserialize::deserialize(&mut deserializer)?;
+        deserializer.end()?;
+        let mut r = HashMap::new();
+        fn collect_from_thread_node(
+            r: &mut HashMap<String, usize>,
+            tn: &ThreadNode,
+        ) -> Result<(), NotmuchError> {
+            let Some(msg) = &tn.0 else {
+                return Ok(());
+            };
+            let mut addrs = vec![];
+            let hdr = &msg.headers.to;
+            if let Some(to) = hdr {
+                addrs.push(to);
+            } else {
+                let hdr = &msg.headers.x_original_to;
+                if let Some(to) = hdr {
+                    addrs.push(to);
+                } else {
+                    let hdr = &msg.headers.delivered_to;
+                    if let Some(to) = hdr {
+                        addrs.push(to);
+                    };
+                };
+            };
+            let hdr = &msg.headers.cc;
+            if let Some(cc) = hdr {
+                addrs.push(cc);
+            };
+            for recipient in addrs {
+                mailparse::addrparse(&recipient)?
+                    .into_inner()
+                    .iter()
+                    .for_each(|a| {
+                        let mailparse::MailAddr::Single(si) = a else {
+                            return;
+                        };
+                        let addr = &si.addr;
+
+                        if addr == "couchmoney@gmail.com" || addr.ends_with("@xinu.tv") {
+                            *r.entry(addr.to_lowercase()).or_default() += 1;
+                        }
+                    });
+            }
+            Ok(())
+        }
+        for t in ts.0 {
+            for tn in t.0 {
+                collect_from_thread_node(&mut r, &tn)?;
+                for sub_tn in tn.1 {
+                    collect_from_thread_node(&mut r, &sub_tn)?;
+                }
+            }
+        }
+        Ok(r)
+    }
+
     fn run_notmuch<I, S>(&self, args: I) -> Result<Vec<u8>, NotmuchError>
     where
         I: IntoIterator<Item = S>,

notmuch/tests/allmail.rs

@@ -1,11 +1,10 @@
 use std::{
     error::Error,
     io::{stdout, Write},
-    time::{Duration, Instant},
+    time::Instant,
 };
 
-use itertools::Itertools;
-use notmuch::{Notmuch, NotmuchError, SearchSummary, ThreadSet};
+use letterbox_notmuch::Notmuch;
 use rayon::iter::{ParallelBridge, ParallelIterator};
 
 #[test]
@@ -23,11 +22,11 @@ fn parse_one() -> Result<(), Box<dyn Error>> {
     let total = nm.count("*")? as f32;
     let start = Instant::now();
     nm.message_ids("*")?
+        .iter()
         .enumerate()
         .par_bridge()
         .for_each(|(i, msg)| {
-            let msg = msg.expect("failed to unwrap msg");
-            let ts = nm
+            let _ts = nm
                 .show(&msg)
                 .expect(&format!("failed to show msg: {}", msg));
             //println!("{:?}", ts);
@@ -77,11 +76,9 @@ fn parse_bulk() -> Result<(), Box<dyn Error>> {
         .into_iter()
         .enumerate()
         //.par_bridge()
-        .for_each(|(i, chunk)| {
-            let msgs: Result<Vec<_>, _> = chunk.collect();
-            let msgs = msgs.expect("failed to unwrap msg");
+        .for_each(|(i, msgs)| {
             let query = msgs.join(" OR ");
-            let ts = nm
+            let _ts = nm
                 .show(&query)
                 .expect(&format!("failed to show msgs: {}", query));
             //println!("{:?}", ts);

procmail2notmuch/Cargo.toml

@@ -1,7 +1,12 @@
 [package]
-name = "procmail2notmuch"
-version = "0.1.0"
-edition = "2021"
+name = "letterbox-procmail2notmuch"
+description = "Tool for generating notmuch rules from procmail"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

scripts/update-crate-version.sh

@@ -0,0 +1,5 @@
+#!env bash
+set -e -x
+cargo-set-version set-version --bump patch
+VERSION="$(awk -F\" '/^version/ {print $2}' server/Cargo.toml)"
+git commit Cargo.lock */Cargo.toml -m "Bumping version to ${VERSION:?}"

server/.sqlx/query-126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\nSELECT\n  url\nFROM email_photo ep\nJOIN email_address ea\nON ep.id = ea.email_photo_id\nWHERE\n  address = $1\n    ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "url",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef"
+}

server/.sqlx/query-2dcbedef656e1b725c5ba4fb67d31ce7962d8714449b2fb630f49a7ed1acc270.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    site,\n    name,\n    count (\n        NOT is_read\n        OR NULL\n    ) unread\nFROM\n    post AS p\n    JOIN feed AS f ON p.site = f.slug --\n    -- TODO: figure this out to make the query faster when only looking for unread\n    --WHERE\n    --    (\n    --        NOT $1\n    --        OR NOT is_read\n    --    )\nGROUP BY\n    1,\n    2\nORDER BY\n    site\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "unread",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true,
+      true,
+      null
+    ]
+  },
+  "hash": "2dcbedef656e1b725c5ba4fb67d31ce7962d8714449b2fb630f49a7ed1acc270"
+}

server/.sqlx/query-383221a94bc3746322ba78e41cde37994440ee67dc32e88d2394c51211bde6cd.json

@@ -0,0 +1,70 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    date,\n    is_read,\n    link,\n    site,\n    summary,\n    clean_summary,\n    title,\n    name,\n    homepage\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug\nWHERE\n    uid = $1\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "date",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_read",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "link",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "clean_summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 8,
+        "name": "homepage",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "383221a94bc3746322ba78e41cde37994440ee67dc32e88d2394c51211bde6cd"
+}

server/.sqlx/query-3d271b404f06497a5dcde68cf6bf07291d70fa56058ea736ac24e91d33050c04.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE\n    search_summary IS NULL\n    -- TODO remove AND link ~ '^<'\nORDER BY\n    ROW_NUMBER() OVER (PARTITION BY site ORDER BY date DESC)\nLIMIT 100;\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "link",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "clean_summary",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "3d271b404f06497a5dcde68cf6bf07291d70fa56058ea736ac24e91d33050c04"
+}

server/.sqlx/query-8c1b3c78649135e98b89092237750088433f7ff1b7c2ddeedec553406ea9f203.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) AS count\nFROM\n    post\nWHERE\n    (\n        $1::text IS NULL\n        OR site = $1\n    )\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n        $3::text IS NULL\n        OR TO_TSVECTOR('english', search_summary)\n        @@ WEBSEARCH_TO_TSQUERY('english', $3)\n    )\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8c1b3c78649135e98b89092237750088433f7ff1b7c2ddeedec553406ea9f203"
+}

server/.sqlx/query-b39147b9d06171cb742141eda4675688cb702fb284758b1224ed3aa2d7f3b3d9.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE\n    post\nSET\n    is_read = $1\nWHERE\n    uid = $2\n",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b39147b9d06171cb742141eda4675688cb702fb284758b1224ed3aa2d7f3b3d9"
+}

server/.sqlx/query-dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\nSELECT id\nFROM feed\nWHERE slug = $1\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc"
+}

server/.sqlx/query-e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\nINSERT INTO feed ( name, slug, url, homepage, selector )\nVALUES ( $1, $2, $3, '', '' )\nRETURNING id\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd"
+}

server/.sqlx/query-ef8327f039dbfa8f4e59b7a77a6411252a346bf51cf940024a17d9fbb2df173c.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE post SET search_summary = $1 WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ef8327f039dbfa8f4e59b7a77a6411252a346bf51cf940024a17d9fbb2df173c"
+}

server/.sqlx/query-fc4607f02cc76a5f3a6629cce4507c74f52ae44820897b47365da3f339d1da06.json

@@ -0,0 +1,56 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $5 :: text IS NULL\n      OR to_tsvector('english', search_summary) @@ websearch_to_tsquery('english', $5)\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "date",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_read",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "uid",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Int8",
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "fc4607f02cc76a5f3a6629cce4507c74f52ae44820897b47365da3f339d1da06"
+}

server/Cargo.toml

@@ -1,30 +1,59 @@
 [package]
-name = "server"
-version = "0.1.0"
-edition = "2021"
-default-bin = "server"
+name = "letterbox-server"
+default-run = "letterbox-server"
+description = "Backend for letterbox"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rocket = { version = "0.5.0-rc.2", features = [ "json" ] }
-notmuch = { path = "../notmuch" }
-shared = { path = "../shared" }
-serde_json = "1.0.87"
-thiserror = "1.0.37"
-serde = { version = "1.0.147", features = ["derive"] }
+ammonia = "4.0.0"
+anyhow = "1.0.79"
+async-graphql = { version = "7", features = ["log"] }
+async-graphql-rocket = "7"
+async-trait = "0.1.81"
+build-info = "0.0.39"
+cacher = { version = "0.1.0", registry = "xinu" }
+chrono = "0.4.39"
+clap = { version = "4.5.23", features = ["derive"] }
+css-inline = "0.14.0"
+futures = "0.3.31"
+html-escape = "0.2.13"
+linkify = "0.10.0"
 log = "0.4.17"
-tokio = "1.26.0"
-glog = "0.1.0"
-urlencoding = "2.1.3"
-async-graphql = { version = "6.0.11", features = ["log"] }
-async-graphql-rocket = "6.0.11"
-rocket_cors = "0.6.0"
+lol_html = "2.0.0"
+mailparse = "0.16.0"
+maplit = "1.0.2"
 memmap = "0.7.0"
-mailparse = "0.14.0"
-ammonia = "3.3.0"
+opentelemetry = "0.28.0"
+regex = "1.11.1"
+reqwest = { version = "0.12.7", features = ["blocking"] }
+rocket = { version = "0.5.0-rc.2", features = ["json"] }
+rocket_cors = "0.6.0"
+scraper = "0.23.0"
+serde = { version = "1.0.147", features = ["derive"] }
+serde_json = "1.0.87"
+sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
+tantivy = { version = "0.22.0", optional = true }
+thiserror = "2.0.0"
+tokio = "1.26.0"
+tracing = "0.1.41"
+url = "2.5.2"
+urlencoding = "2.1.3"
+#xtracing = { path = "../../xtracing" }
+#xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
+xtracing = { version = "0.3.0", registry = "xinu" }
+letterbox-notmuch = { version = "0.9.3", path = "../notmuch", registry = "xinu" }
+letterbox-shared = { version = "0.9.3", path = "../shared", registry = "xinu" }
 
-[dependencies.rocket_contrib]
-version = "0.4.11"
-default-features = false
-features = ["json"]
+[build-dependencies]
+build-info-build = "0.0.39"
+
+[features]
+#default = [ "tantivy" ]
+tantivy = ["dep:tantivy"]

server/Rocket.toml

@@ -1,9 +1,14 @@
 [release]
 address = "0.0.0.0"
 port = 9345
+newsreader_database_url = "postgres://newsreader@nixos-07.h.xinu.tv/newsreader"
+newsreader_tantivy_db_path = "../target/database/newsreader"
 
 [debug]
 address = "0.0.0.0"
 port = 9345
 # Uncomment to make it production like.
 #log_level = "critical"
+newsreader_database_url = "postgres://newsreader@nixos-07.h.xinu.tv/newsreader"
+newsreader_tantivy_db_path = "../target/database/newsreader"
+slurp_cache_path = "/tmp/letterbox/slurp"

server/build.rs

@@ -0,0 +1,5 @@
+fn main() {
+    // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
+    // and `build_info::format!` in the main program.
+    build_info_build::build_script();
+}

server/migrations/20241217021645_create-search-table.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS post_summary_idx;
+DROP INDEX IF EXISTS post_site_idx;
+DROP INDEX IF EXISTS post_title_idx;

server/migrations/20241217021645_create-search-table.up.sql

@@ -0,0 +1,3 @@
+CREATE INDEX post_summary_idx ON post USING GIN (to_tsvector('english', summary));
+CREATE INDEX post_site_idx ON post USING GIN (to_tsvector('english', site));
+CREATE INDEX post_title_idx ON post USING GIN (to_tsvector('english', title));

server/migrations/20241218010438_create-email-tables.down.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+ALTER TABLE IF EXISTS public."Email" DROP CONSTRAINT IF EXISTS email_avatar_fkey;
+ALTER TABLE IF EXISTS public."EmailDisplayName" DROP CONSTRAINT IF EXISTS email_id_fk;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_to_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_cc_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_from_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_header_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_file_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_body_id_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_thread_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_tag_fkey;
+
+DROP TABLE IF EXISTS public."Email";
+DROP TABLE IF EXISTS public."EmailDisplayName";
+DROP TABLE IF EXISTS public."Message";
+DROP TABLE IF EXISTS public."Header";
+DROP TABLE IF EXISTS public."File";
+DROP TABLE IF EXISTS public."Avatar";
+DROP TABLE IF EXISTS public."Body";
+DROP TABLE IF EXISTS public."Thread";
+DROP TABLE IF EXISTS public."Tag";
+
+END;

server/migrations/20241218010438_create-email-tables.up.sql

@@ -0,0 +1,174 @@
+-- This script was generated by the ERD tool in pgAdmin 4.
+-- Please log an issue at https://github.com/pgadmin-org/pgadmin4/issues/new/choose if you find any bugs, including reproduction steps.
+BEGIN;
+
+ALTER TABLE IF EXISTS public."Email" DROP CONSTRAINT IF EXISTS email_avatar_fkey;
+ALTER TABLE IF EXISTS public."EmailDisplayName" DROP CONSTRAINT IF EXISTS email_id_fk;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_to_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_cc_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_from_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_header_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_file_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_body_id_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_thread_fkey;
+ALTER TABLE IF EXISTS public."Message" DROP CONSTRAINT IF EXISTS message_tag_fkey;
+
+CREATE TABLE IF NOT EXISTS public."Email"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    address text NOT NULL,
+    avatar_id integer,
+    PRIMARY KEY (id),
+    CONSTRAINT avatar_id UNIQUE (avatar_id)
+);
+
+CREATE TABLE IF NOT EXISTS public."EmailDisplayName"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    email_id integer NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Message"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    subject text,
+    "from" integer,
+    "to" integer,
+    cc integer,
+    header_id integer,
+    hash text NOT NULL,
+    file_id integer NOT NULL,
+    date timestamp with time zone NOT NULL,
+    unread boolean NOT NULL,
+    body_id integer NOT NULL,
+    thread_id integer NOT NULL,
+    tag_id integer,
+    CONSTRAINT body_id UNIQUE (body_id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Header"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    key text NOT NULL,
+    value text NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."File"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    path text NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Avatar"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    url text NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Body"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    text text NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Thread"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS public."Tag"
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    name text NOT NULL,
+    display text,
+    fg_color integer,
+    bg_color integer,
+    PRIMARY KEY (id)
+);
+
+ALTER TABLE IF EXISTS public."Email"
+    ADD CONSTRAINT email_avatar_fkey FOREIGN KEY (avatar_id)
+    REFERENCES public."Avatar" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."EmailDisplayName"
+    ADD CONSTRAINT email_id_fk FOREIGN KEY (email_id)
+    REFERENCES public."Email" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_to_fkey FOREIGN KEY ("to")
+    REFERENCES public."Email" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_cc_fkey FOREIGN KEY (cc)
+    REFERENCES public."Email" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_from_fkey FOREIGN KEY ("from")
+    REFERENCES public."Email" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_header_fkey FOREIGN KEY (header_id)
+    REFERENCES public."Header" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_file_fkey FOREIGN KEY (file_id)
+    REFERENCES public."File" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_body_id_fkey FOREIGN KEY (body_id)
+    REFERENCES public."Body" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_thread_fkey FOREIGN KEY (thread_id)
+    REFERENCES public."Thread" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+
+ALTER TABLE IF EXISTS public."Message"
+    ADD CONSTRAINT message_tag_fkey FOREIGN KEY (tag_id)
+    REFERENCES public."Tag" (id) MATCH SIMPLE
+    ON UPDATE NO ACTION
+    ON DELETE NO ACTION
+    NOT VALID;
+
+END;

server/migrations/20241225054216_post-link-unique.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE
+    post DROP CONSTRAINT post_link_key;

server/migrations/20241225054216_post-link-unique.up.sql

@@ -0,0 +1,28 @@
+WITH dupes AS (
+    SELECT
+        uid,
+        link,
+        Row_number() over(
+            PARTITION by link
+            ORDER BY
+                link
+        ) AS RowNumber
+    FROM
+        post
+)
+DELETE FROM
+    post
+WHERE
+    uid IN (
+        SELECT
+            uid
+        FROM
+            dupes
+        WHERE
+            RowNumber > 1
+    );
+
+ALTER TABLE
+    post
+ADD
+    UNIQUE (link);

server/migrations/20241225180250_post-link-non-empty.down.sql

@@ -0,0 +1,7 @@
+ALTER TABLE
+    post
+ALTER COLUMN
+    link DROP NOT NULL;
+
+ALTER TABLE
+    post DROP CONSTRAINT link;

server/migrations/20241225180250_post-link-non-empty.up.sql

@@ -0,0 +1,17 @@
+DELETE FROM
+    post
+WHERE
+    link IS NULL
+    OR link = '';
+
+ALTER TABLE
+    post
+ALTER COLUMN
+    link
+SET
+    NOT NULL;
+
+ALTER TABLE
+    post
+ADD
+    CONSTRAINT link CHECK (link <> '');

server/migrations/20250114045217_create-email-photos.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS email_address;
+DROP TABLE IF EXISTS photo;
+DROP TABLE IF EXISTS google_person;

server/migrations/20250114045217_create-email-photos.up.sql

@@ -0,0 +1,19 @@
+-- Add up migration script here
+CREATE TABLE IF NOT EXISTS google_person (
+  id SERIAL PRIMARY KEY,
+  resource_name TEXT NOT NULL UNIQUE,
+  display_name TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS email_photo (
+  id SERIAL PRIMARY KEY,
+  google_person_id INTEGER REFERENCES google_person (id) UNIQUE,
+  url TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS email_address (
+  id SERIAL PRIMARY KEY,
+  address TEXT NOT NULL UNIQUE,
+  email_photo_id INTEGER REFERENCES email_photo (id),
+  google_person_id INTEGER REFERENCES google_person (id)
+);

server/migrations/20250117225159_strip-html-index.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+DROP INDEX post_summary_idx;
+CREATE INDEX post_summary_idx ON post USING gin (
+    to_tsvector('english', summary)
+);

server/migrations/20250117225159_strip-html-index.up.sql

@@ -0,0 +1,11 @@
+-- Something like this around summary in the idx w/ tsvector
+DROP INDEX post_summary_idx;
+CREATE INDEX post_summary_idx ON post USING gin (to_tsvector(
+    'english',
+    regexp_replace(
+        regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+        '\s+',
+        ' ',
+        'g'
+    )
+));

server/migrations/20250127192339_nzb_posts_created_at_idx.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP INDEX nzb_posts_created_at_idx;

server/migrations/20250127192339_nzb_posts_created_at_idx.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX nzb_posts_created_at_idx ON nzb_posts USING btree (created_at);

server/migrations/20250128234348_add-search-summary.down.sql

@@ -0,0 +1,15 @@
+-- Add down migration script here
+BEGIN;
+DROP INDEX IF EXISTS post_search_summary_idx;
+ALTER TABLE post DROP search_summary;
+
+-- CREATE INDEX post_summary_idx ON post USING gin (to_tsvector(
+--     'english',
+--     regexp_replace(
+--         regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+--         '\s+',
+--         ' ',
+--         'g'
+--     )
+-- ));
+COMMIT;

server/migrations/20250128234348_add-search-summary.up.sql

@@ -0,0 +1,14 @@
+-- Add up migration script here
+BEGIN;
+DROP INDEX IF EXISTS post_summary_idx;
+ALTER TABLE post ADD search_summary TEXT;
+CREATE INDEX post_search_summary_idx ON post USING gin (
+    to_tsvector('english', search_summary)
+);
+UPDATE post SET search_summary = regexp_replace(
+    regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+    '\s+',
+    ' ',
+    'g'
+);
+COMMIT;

server/sql/all-posts.sql

@@ -0,0 +1,14 @@
+SELECT
+    site,
+    title,
+    summary,
+    link,
+    date,
+    is_read,
+    uid,
+    p.id id
+FROM
+    post AS p
+    JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
+ORDER BY
+    date DESC;

server/sql/all-uids.sql

@@ -0,0 +1,6 @@
+SELECT
+    uid
+FROM
+    post AS p
+    JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
+;

server/sql/count.sql

@@ -0,0 +1,17 @@
+SELECT COUNT(*) AS count
+FROM
+    post
+WHERE
+    (
+        $1::text IS NULL
+        OR site = $1
+    )
+    AND (
+        NOT $2
+        OR NOT is_read
+    )
+    AND (
+        $3::text IS NULL
+        OR TO_TSVECTOR('english', search_summary)
+        @@ WEBSEARCH_TO_TSQUERY('english', $3)
+    )

server/sql/need-search-summary.sql

@@ -0,0 +1,13 @@
+SELECT
+    p.id,
+    link,
+    clean_summary
+FROM
+    post AS p
+INNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
+WHERE
+    search_summary IS NULL
+    -- TODO remove AND link ~ '^<'
+ORDER BY
+    ROW_NUMBER() OVER (PARTITION BY site ORDER BY date DESC)
+LIMIT 100;

server/sql/posts-from-uids.sql

@@ -0,0 +1,14 @@
+SELECT
+    site AS "site!",
+    title AS "title!",
+    summary AS "summary!",
+    link AS "link!",
+    date AS "date!",
+    is_read AS "is_read!",
+    uid AS "uid!",
+    p.id id
+FROM
+    post p
+    JOIN feed f ON p.site = f.slug
+WHERE
+    uid = ANY ($1);

server/sql/set_unread.sql

@@ -0,0 +1,6 @@
+UPDATE
+    post
+SET
+    is_read = $1
+WHERE
+    uid = $2

server/sql/tags.sql

@@ -0,0 +1,21 @@
+SELECT
+    site,
+    name,
+    count (
+        NOT is_read
+        OR NULL
+    ) unread
+FROM
+    post AS p
+    JOIN feed AS f ON p.site = f.slug --
+    -- TODO: figure this out to make the query faster when only looking for unread
+    --WHERE
+    --    (
+    --        NOT $1
+    --        OR NOT is_read
+    --    )
+GROUP BY
+    1,
+    2
+ORDER BY
+    site

server/sql/thread.sql

@@ -0,0 +1,15 @@
+SELECT
+    date,
+    is_read,
+    link,
+    site,
+    summary,
+    clean_summary,
+    title,
+    name,
+    homepage
+FROM
+    post AS p
+INNER JOIN feed AS f ON p.site = f.slug
+WHERE
+    uid = $1

server/sql/threads-from-uid.sql

@@ -0,0 +1,14 @@
+SELECT
+    site,
+    date,
+    is_read,
+    title,
+    uid,
+    name
+FROM
+    post p
+    JOIN feed f ON p.site = f.slug
+WHERE
+    uid = ANY ($1)
+ORDER BY
+    date DESC;

server/sql/threads.sql

@@ -0,0 +1,25 @@
+SELECT
+    site,
+    date,
+    is_read,
+    title,
+    uid,
+    name
+FROM
+    post p
+    JOIN feed f ON p.site = f.slug
+WHERE
+    ($1::text IS NULL OR site = $1)
+    AND (
+        NOT $2
+        OR NOT is_read
+    )
+    AND (
+      $5 :: text IS NULL
+      OR to_tsvector('english', search_summary) @@ websearch_to_tsquery('english', $5)
+    )
+ORDER BY
+    date DESC,
+    title OFFSET $3
+LIMIT
+    $4

server/sql/too-long-for-search.sql

@@ -0,0 +1,13 @@
+select t.id, tt.tokid, tt.alias, length(t.token), t.token from (
+    select  id, (ts_parse('default', 
+    -- regexp_replace(
+    --     regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+    --     '\s+',
+    --     ' ',
+    --     'g'
+    -- )
+        summary
+    )).* from post) t
+    inner join ts_token_type('default') tt
+       on t.tokid = tt.tokid
+    where length(token) >= 2*1024;

server/src/bin/cleanhtml.rs

@@ -0,0 +1,16 @@
+use std::fs;
+
+use letterbox_server::sanitize_html;
+
+fn main() -> anyhow::Result<()> {
+    let mut args = std::env::args().skip(1);
+    let src = args.next().expect("source not specified");
+    let dst = args.next().expect("destination not specified");
+    println!("Sanitizing {src} into {dst}");
+    let bytes = fs::read(src)?;
+    let html = String::from_utf8_lossy(&bytes);
+    let html = sanitize_html(&html, "", &None)?;
+    fs::write(dst, html)?;
+
+    Ok(())
+}

server/src/bin/email2db.rs

@@ -0,0 +1,22 @@
+use clap::Parser;
+use letterbox_server::mail::read_mail_to_db;
+use sqlx::postgres::PgPool;
+
+/// Add certain emails as posts in newsfeed app.
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    /// DB URL, something like postgres://newsreader@nixos-07.h.xinu.tv/newsreader
+    #[arg(short, long)]
+    db_url: String,
+    /// path to parse
+    path: String,
+}
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
+    let args = Args::parse();
+    let pool = PgPool::connect(&args.db_url).await?;
+    read_mail_to_db(&pool, &args.path).await?;
+    Ok(())
+}

server/src/bin/inline_html.rs

@@ -0,0 +1,21 @@
+use std::fs;
+
+use url::Url;
+
+fn main() -> anyhow::Result<()> {
+    println!("PWD: {}", std::env::current_dir()?.display());
+    let _url = "https://slashdot.org/story/25/01/24/1813201/walgreens-replaced-fridge-doors-with-smart-screens-its-now-a-200-million-fiasco?utm_source=rss1.0mainlinkanon&utm_medium=feed";
+    let _url = "https://hackaday.com/2025/01/24/hackaday-podcast-episode-305-caustic-clocks-practice-bones-and-brick-layers/";
+    let _url =    "https://theonion.com/monster-devastated-to-see-film-depicting-things-he-told-guillermo-del-toro-in-confidence/";
+    let _url = "https://trofi.github.io/posts/330-another-nix-language-nondeterminism-example.html";
+    let _url = "https://blog.cloudflare.com/ddos-threat-report-for-2024-q4/";
+    let url = "https://trofi.github.io/posts/330-another-nix-language-nondeterminism-example.html";
+    let body = reqwest::blocking::get(url)?.text()?;
+    let output = "/tmp/h2md/output.html";
+    let inliner = css_inline::CSSInliner::options()
+        .base_url(Url::parse(url).ok())
+        .build();
+    let inlined = inliner.inline(&body)?;
+    fs::write(output, inlined)?;
+    Ok(())
+}

server/src/bin/letterbox-server.rs

@@ -0,0 +1,240 @@
+// Rocket generates a lot of warnings for handlers
+// TODO: figure out why
+#![allow(unreachable_patterns)]
+#[macro_use]
+extern crate rocket;
+use std::{error::Error, io::Cursor, str::FromStr};
+
+use async_graphql::{extensions, http::GraphiQLSource, EmptySubscription, Schema};
+use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
+use cacher::FilesystemCacher;
+use letterbox_notmuch::{Notmuch, NotmuchError, ThreadSet};
+#[cfg(feature = "tantivy")]
+use letterbox_server::tantivy::TantivyConnection;
+use letterbox_server::{
+    config::Config,
+    error::ServerError,
+    graphql::{Attachment, GraphqlSchema, Mutation, QueryRoot},
+    nm::{attachment_bytes, cid_attachment_bytes},
+};
+use rocket::{
+    fairing::AdHoc,
+    http::{ContentType, Header},
+    request::Request,
+    response::{content, Debug, Responder},
+    serde::json::Json,
+    Response, State,
+};
+use rocket_cors::{AllowedHeaders, AllowedOrigins};
+use sqlx::postgres::PgPool;
+
+#[get("/show/<query>/pretty")]
+async fn show_pretty(
+    nm: &State<Notmuch>,
+    query: &str,
+) -> Result<Json<ThreadSet>, Debug<ServerError>> {
+    let query = urlencoding::decode(query).map_err(|e| ServerError::from(NotmuchError::from(e)))?;
+    let res = nm.show(&query).map_err(ServerError::from)?;
+    Ok(Json(res))
+}
+
+#[get("/show/<query>")]
+async fn show(nm: &State<Notmuch>, query: &str) -> Result<Json<ThreadSet>, Debug<NotmuchError>> {
+    let query = urlencoding::decode(query).map_err(NotmuchError::from)?;
+    let res = nm.show(&query)?;
+    Ok(Json(res))
+}
+
+struct InlineAttachmentResponder(Attachment);
+
+impl<'r, 'o: 'r> Responder<'r, 'o> for InlineAttachmentResponder {
+    fn respond_to(self, _: &'r Request<'_>) -> rocket::response::Result<'o> {
+        let mut resp = Response::build();
+        if let Some(filename) = self.0.filename {
+            resp.header(Header::new(
+                "Content-Disposition",
+                format!(r#"inline; filename="{}""#, filename),
+            ));
+        }
+        if let Some(content_type) = self.0.content_type {
+            if let Some(ct) = ContentType::parse_flexible(&content_type) {
+                resp.header(ct);
+            }
+        }
+        resp.sized_body(self.0.bytes.len(), Cursor::new(self.0.bytes))
+            .ok()
+    }
+}
+
+struct DownloadAttachmentResponder(Attachment);
+
+impl<'r, 'o: 'r> Responder<'r, 'o> for DownloadAttachmentResponder {
+    fn respond_to(self, _: &'r Request<'_>) -> rocket::response::Result<'o> {
+        let mut resp = Response::build();
+        if let Some(filename) = self.0.filename {
+            resp.header(Header::new(
+                "Content-Disposition",
+                format!(r#"attachment; filename="{}""#, filename),
+            ));
+        }
+        if let Some(content_type) = self.0.content_type {
+            if let Some(ct) = ContentType::parse_flexible(&content_type) {
+                resp.header(ct);
+            }
+        }
+        resp.sized_body(self.0.bytes.len(), Cursor::new(self.0.bytes))
+            .ok()
+    }
+}
+
+#[get("/cid/<id>/<cid>")]
+async fn view_cid(
+    nm: &State<Notmuch>,
+    id: &str,
+    cid: &str,
+) -> Result<InlineAttachmentResponder, Debug<ServerError>> {
+    let mid = if id.starts_with("id:") {
+        id.to_string()
+    } else {
+        format!("id:{}", id)
+    };
+    info!("view cid attachment {mid} {cid}");
+    let attachment = cid_attachment_bytes(nm, &mid, &cid)?;
+    Ok(InlineAttachmentResponder(attachment))
+}
+
+#[get("/view/attachment/<id>/<idx>/<_>")]
+async fn view_attachment(
+    nm: &State<Notmuch>,
+    id: &str,
+    idx: &str,
+) -> Result<InlineAttachmentResponder, Debug<ServerError>> {
+    let mid = if id.starts_with("id:") {
+        id.to_string()
+    } else {
+        format!("id:{}", id)
+    };
+    info!("view attachment {mid} {idx}");
+    let idx: Vec<_> = idx
+        .split('.')
+        .map(|s| s.parse().expect("not a usize"))
+        .collect();
+    let attachment = attachment_bytes(nm, &mid, &idx)?;
+    Ok(InlineAttachmentResponder(attachment))
+}
+
+#[get("/download/attachment/<id>/<idx>/<_>")]
+async fn download_attachment(
+    nm: &State<Notmuch>,
+    id: &str,
+    idx: &str,
+) -> Result<DownloadAttachmentResponder, Debug<ServerError>> {
+    let mid = if id.starts_with("id:") {
+        id.to_string()
+    } else {
+        format!("id:{}", id)
+    };
+    info!("download attachment {mid} {idx}");
+    let idx: Vec<_> = idx
+        .split('.')
+        .map(|s| s.parse().expect("not a usize"))
+        .collect();
+    let attachment = attachment_bytes(nm, &mid, &idx)?;
+    Ok(DownloadAttachmentResponder(attachment))
+}
+
+#[get("/original/<id>")]
+async fn original(
+    nm: &State<Notmuch>,
+    id: &str,
+) -> Result<(ContentType, Vec<u8>), Debug<NotmuchError>> {
+    let mid = if id.starts_with("id:") {
+        id.to_string()
+    } else {
+        format!("id:{}", id)
+    };
+    let res = nm.show_original(&mid)?;
+    Ok((ContentType::Plain, res))
+}
+
+#[rocket::get("/")]
+fn graphiql() -> content::RawHtml<String> {
+    content::RawHtml(GraphiQLSource::build().endpoint("/api/graphql").finish())
+}
+
+#[rocket::get("/graphql?<query..>")]
+async fn graphql_query(schema: &State<GraphqlSchema>, query: GraphQLQuery) -> GraphQLResponse {
+    query.execute(schema.inner()).await
+}
+
+#[rocket::post("/graphql", data = "<request>", format = "application/json")]
+async fn graphql_request(
+    schema: &State<GraphqlSchema>,
+    request: GraphQLRequest,
+) -> GraphQLResponse {
+    request.execute(schema.inner()).await
+}
+
+#[rocket::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
+    build_info::build_info!(fn bi);
+    info!("Build Info: {}", letterbox_shared::build_version(bi));
+    let allowed_origins = AllowedOrigins::all();
+    let cors = rocket_cors::CorsOptions {
+        allowed_origins,
+        allowed_methods: vec!["Get"]
+            .into_iter()
+            .map(|s| FromStr::from_str(s).unwrap())
+            .collect(),
+        allowed_headers: AllowedHeaders::some(&["Authorization", "Accept"]),
+        allow_credentials: true,
+        ..Default::default()
+    }
+    .to_cors()?;
+
+    let rkt = rocket::build()
+        .mount(
+            letterbox_shared::urls::MOUNT_POINT,
+            routes![
+                original,
+                show_pretty,
+                show,
+                graphql_query,
+                graphql_request,
+                graphiql,
+                view_cid,
+                view_attachment,
+                download_attachment,
+            ],
+        )
+        .attach(cors)
+        .attach(AdHoc::config::<Config>());
+
+    let config: Config = rkt.figment().extract()?;
+    if !std::fs::exists(&config.slurp_cache_path)? {
+        info!("Creating slurp cache @ '{}'", &config.slurp_cache_path);
+        std::fs::create_dir_all(&config.slurp_cache_path)?;
+    }
+    let pool = PgPool::connect(&config.newsreader_database_url).await?;
+    sqlx::migrate!("./migrations").run(&pool).await?;
+    #[cfg(feature = "tantivy")]
+    let tantivy_conn = TantivyConnection::new(&config.newsreader_tantivy_db_path)?;
+
+    let cacher = FilesystemCacher::new(&config.slurp_cache_path)?;
+    let schema = Schema::build(QueryRoot, Mutation, EmptySubscription)
+        .data(Notmuch::default())
+        .data(cacher)
+        .data(pool.clone());
+
+    #[cfg(feature = "tantivy")]
+    let schema = schema.data(tantivy_conn);
+
+    let schema = schema.extension(extensions::Logger).finish();
+
+    let rkt = rkt.manage(schema).manage(pool).manage(Notmuch::default());
+    //.manage(Notmuch::with_config("../notmuch/testdata/notmuch.config"))
+
+    rkt.launch().await?;
+    Ok(())
+}

server/src/bin/server.rs

@@ -1,215 +0,0 @@
-#[macro_use]
-extern crate rocket;
-use std::{error::Error, io::Cursor, str::FromStr};
-
-use async_graphql::{http::GraphiQLSource, EmptyMutation, EmptySubscription, Schema};
-use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
-use glog::Flags;
-use notmuch::{Notmuch, NotmuchError, ThreadSet};
-use rocket::{
-    http::{ContentType, Header},
-    request::Request,
-    response::{content, Debug, Responder},
-    serde::json::Json,
-    Response, State,
-};
-use rocket_cors::{AllowedHeaders, AllowedOrigins};
-use server::{
-    error::ServerError,
-    graphql::{GraphqlSchema, QueryRoot},
-};
-
-#[get("/refresh")]
-async fn refresh(nm: &State<Notmuch>) -> Result<Json<String>, Debug<NotmuchError>> {
-    Ok(Json(String::from_utf8_lossy(&nm.new()?).to_string()))
-}
-
-#[get("/search")]
-async fn search_all(
-    nm: &State<Notmuch>,
-) -> Result<Json<shared::SearchResult>, Debug<NotmuchError>> {
-    search(nm, "*", None, None).await
-}
-
-#[get("/search/<query>?<page>&<results_per_page>")]
-async fn search(
-    nm: &State<Notmuch>,
-    query: &str,
-    page: Option<usize>,
-    results_per_page: Option<usize>,
-) -> Result<Json<shared::SearchResult>, Debug<NotmuchError>> {
-    let page = page.unwrap_or(0);
-    let results_per_page = results_per_page.unwrap_or(20);
-    let query = urlencoding::decode(query).map_err(NotmuchError::from)?;
-    info!(" search '{query}'");
-    let res = shared::SearchResult {
-        summary: nm.search(&query, page * results_per_page, results_per_page)?,
-        query: query.to_string(),
-        page,
-        results_per_page,
-        total: nm.count(&query)?,
-    };
-    Ok(Json(res))
-}
-
-#[get("/show/<query>/pretty")]
-async fn show_pretty(
-    nm: &State<Notmuch>,
-    query: &str,
-) -> Result<Json<ThreadSet>, Debug<ServerError>> {
-    let query = urlencoding::decode(query).map_err(|e| ServerError::from(NotmuchError::from(e)))?;
-    let res = nm.show(&query).map_err(ServerError::from)?;
-    Ok(Json(res))
-}
-
-#[get("/show/<query>")]
-async fn show(nm: &State<Notmuch>, query: &str) -> Result<Json<ThreadSet>, Debug<NotmuchError>> {
-    let query = urlencoding::decode(query).map_err(NotmuchError::from)?;
-    let res = nm.show(&query)?;
-    Ok(Json(res))
-}
-
-struct PartResponder {
-    bytes: Vec<u8>,
-    filename: Option<String>,
-}
-
-impl<'r, 'o: 'r> Responder<'r, 'o> for PartResponder {
-    fn respond_to(self, _: &'r Request<'_>) -> rocket::response::Result<'o> {
-        let mut resp = Response::build();
-        if let Some(filename) = self.filename {
-            info!("filename {:?}", filename);
-            resp.header(Header::new(
-                "Content-Disposition",
-                format!(r#"attachment; filename="{}""#, filename),
-            ))
-            .header(ContentType::Binary);
-        }
-        resp.sized_body(self.bytes.len(), Cursor::new(self.bytes))
-            .ok()
-    }
-}
-
-#[get("/attachment/<id>/<idx>")]
-async fn attachment(
-    _nm: &State<Notmuch>,
-    id: &str,
-    idx: usize,
-) -> Result<PartResponder, Debug<NotmuchError>> {
-    let _idx = idx;
-    let _mid = if id.starts_with("id:") {
-        id.to_string()
-    } else {
-        format!("id:{}", id)
-    };
-    let bytes = Vec::new();
-    let filename = None;
-    // TODO(wathiede): use walk_attachments from graphql to fill this out
-    Ok(PartResponder { bytes, filename })
-}
-
-#[get("/original/<id>/part/<part>")]
-async fn original_part(
-    nm: &State<Notmuch>,
-    id: &str,
-    part: usize,
-) -> Result<PartResponder, Debug<NotmuchError>> {
-    let mid = if id.starts_with("id:") {
-        id.to_string()
-    } else {
-        format!("id:{}", id)
-    };
-    let meta = nm.show_part(&mid, part)?;
-    let res = nm.show_original_part(&mid, part)?;
-    Ok(PartResponder {
-        bytes: res,
-        filename: meta.filename,
-    })
-}
-
-#[get("/original/<id>")]
-async fn original(
-    nm: &State<Notmuch>,
-    id: &str,
-) -> Result<(ContentType, Vec<u8>), Debug<NotmuchError>> {
-    let mid = if id.starts_with("id:") {
-        id.to_string()
-    } else {
-        format!("id:{}", id)
-    };
-    let res = nm.show_original(&mid)?;
-    Ok((ContentType::Plain, res))
-}
-
-#[rocket::get("/")]
-fn graphiql() -> content::RawHtml<String> {
-    content::RawHtml(GraphiQLSource::build().endpoint("/graphql").finish())
-}
-
-#[rocket::get("/graphql?<query..>")]
-async fn graphql_query(schema: &State<GraphqlSchema>, query: GraphQLQuery) -> GraphQLResponse {
-    query.execute(schema.inner()).await
-}
-
-#[rocket::post("/graphql", data = "<request>", format = "application/json")]
-async fn graphql_request(
-    schema: &State<GraphqlSchema>,
-    request: GraphQLRequest,
-) -> GraphQLResponse {
-    request.execute(schema.inner()).await
-}
-
-#[rocket::main]
-async fn main() -> Result<(), Box<dyn Error>> {
-    glog::new()
-        .init(Flags {
-            colorlogtostderr: true,
-            //alsologtostderr: true, // use logtostderr to only write to stderr and not to files
-            logtostderr: true,
-            ..Default::default()
-        })
-        .unwrap();
-    let allowed_origins = AllowedOrigins::all();
-    let cors = rocket_cors::CorsOptions {
-        allowed_origins,
-        allowed_methods: vec!["Get"]
-            .into_iter()
-            .map(|s| FromStr::from_str(s).unwrap())
-            .collect(),
-        allowed_headers: AllowedHeaders::some(&["Authorization", "Accept"]),
-        allow_credentials: true,
-        ..Default::default()
-    }
-    .to_cors()?;
-
-    let schema = Schema::build(QueryRoot, EmptyMutation, EmptySubscription)
-        .data(Notmuch::default())
-        .extension(async_graphql::extensions::Logger)
-        .finish();
-
-    let _ = rocket::build()
-        .mount(
-            "/",
-            routes![
-                original_part,
-                original,
-                refresh,
-                search_all,
-                search,
-                show_pretty,
-                show,
-                graphql_query,
-                graphql_request,
-                graphiql,
-                attachment
-            ],
-        )
-        .attach(cors)
-        .manage(schema)
-        .manage(Notmuch::default())
-        //.manage(Notmuch::with_config("../notmuch/testdata/notmuch.config"))
-        .launch()
-        .await?;
-
-    Ok(())
-}

server/src/config.rs

@@ -0,0 +1,7 @@
+use serde::Deserialize;
+#[derive(Deserialize)]
+pub struct Config {
+    pub newsreader_database_url: String,
+    pub newsreader_tantivy_db_path: String,
+    pub slurp_cache_path: String,
+}

server/src/error.rs

@@ -1,9 +1,42 @@
+use std::{convert::Infallible, str::Utf8Error, string::FromUtf8Error};
+
+use mailparse::MailParseError;
+#[cfg(feature = "tantivy")]
+use tantivy::{query::QueryParserError, TantivyError};
 use thiserror::Error;
 
+use crate::TransformError;
+
 #[derive(Error, Debug)]
 pub enum ServerError {
-    #[error("notmuch")]
-    NotmuchError(#[from] notmuch::NotmuchError),
+    #[error("notmuch: {0}")]
+    NotmuchError(#[from] letterbox_notmuch::NotmuchError),
     #[error("flatten")]
     FlattenError,
+    #[error("mail parse error: {0}")]
+    MailParseError(#[from] MailParseError),
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("attachement not found")]
+    PartNotFound,
+    #[error("sqlx error: {0}")]
+    SQLXError(#[from] sqlx::Error),
+    #[error("html transform error: {0}")]
+    TransformError(#[from] TransformError),
+    #[error("UTF8 error: {0}")]
+    Utf8Error(#[from] Utf8Error),
+    #[error("FromUTF8 error: {0}")]
+    FromUtf8Error(#[from] FromUtf8Error),
+    #[error("error: {0}")]
+    StringError(String),
+    #[error("invalid url: {0}")]
+    UrlParseError(#[from] url::ParseError),
+    #[cfg(feature = "tantivy")]
+    #[error("tantivy error: {0}")]
+    TantivyError(#[from] TantivyError),
+    #[cfg(feature = "tantivy")]
+    #[error("tantivy query parse error: {0}")]
+    QueryParseError(#[from] QueryParserError),
+    #[error("impossible: {0}")]
+    InfaillibleError(#[from] Infallible),
 }

server/src/lib.rs

@@ -1,3 +1,912 @@
+pub mod config;
 pub mod error;
 pub mod graphql;
+pub mod mail;
+pub mod newsreader;
 pub mod nm;
+#[cfg(feature = "tantivy")]
+pub mod tantivy;
+
+use std::{
+    collections::{HashMap, HashSet},
+    convert::Infallible,
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
+
+use async_trait::async_trait;
+use cacher::{Cacher, FilesystemCacher};
+use css_inline::{CSSInliner, InlineError, InlineOptions};
+use linkify::{LinkFinder, LinkKind};
+use log::{debug, error, info, warn};
+use lol_html::{
+    element, errors::RewritingError, html_content::ContentType, rewrite_str, text,
+    RewriteStrSettings,
+};
+use maplit::{hashmap, hashset};
+use regex::Regex;
+use reqwest::StatusCode;
+use scraper::{Html, Selector};
+use sqlx::types::time::PrimitiveDateTime;
+use thiserror::Error;
+use url::Url;
+
+use crate::{
+    error::ServerError,
+    graphql::{Corpus, ThreadSummary},
+    newsreader::is_newsreader_thread,
+    nm::is_notmuch_thread_or_id,
+};
+
+const NEWSREADER_TAG_PREFIX: &'static str = "News/";
+const NEWSREADER_THREAD_PREFIX: &'static str = "news:";
+
+// TODO: figure out how to use Cow
+#[async_trait]
+trait Transformer: Send + Sync {
+    fn should_run(&self, _addr: &Option<Url>, _html: &str) -> bool {
+        true
+    }
+    // TODO: should html be something like `html_escape` uses:
+    // <S: ?Sized + AsRef<str>>(text: &S) -> Cow<str>
+    async fn transform(&self, addr: &Option<Url>, html: &str) -> Result<String, TransformError>;
+}
+
+// TODO: how would we make this more generic to allow good implementations of Transformer outside
+// of this module?
+#[derive(Error, Debug)]
+pub enum TransformError {
+    #[error("lol-html rewrite error: {0}")]
+    RewritingError(#[from] RewritingError),
+    #[error("css inline error: {0}")]
+    InlineError(#[from] InlineError),
+    #[error("failed to fetch url error: {0}")]
+    ReqwestError(#[from] reqwest::Error),
+    #[error("failed to parse HTML: {0}")]
+    HtmlParsingError(String),
+    #[error("got a retryable error code {0} for {1}")]
+    RetryableHttpStatusError(StatusCode, String),
+}
+
+struct SanitizeHtml<'a> {
+    cid_prefix: &'a str,
+    base_url: &'a Option<Url>,
+}
+
+#[async_trait]
+impl<'a> Transformer for SanitizeHtml<'a> {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        Ok(sanitize_html(html, self.cid_prefix, self.base_url)?)
+    }
+}
+
+struct EscapeHtml;
+
+#[async_trait]
+impl Transformer for EscapeHtml {
+    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+        html.contains("&")
+    }
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        Ok(html_escape::decode_html_entities(html).to_string())
+    }
+}
+
+struct StripHtml;
+
+#[async_trait]
+impl Transformer for StripHtml {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("StripHtml should_run {link:?} {}", html.contains("<"));
+        // Lame test
+        html.contains("<")
+    }
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("StripHtml {link:?}");
+        let mut text = String::new();
+        let element_content_handlers = vec![
+            element!("style", |el| {
+                el.remove();
+                Ok(())
+            }),
+            element!("script", |el| {
+                el.remove();
+                Ok(())
+            }),
+        ];
+        let html = rewrite_str(
+            html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
+        let element_content_handlers = vec![text!("*", |t| {
+            text += t.as_str();
+            Ok(())
+        })];
+        let _ = rewrite_str(
+            &html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
+        let re = Regex::new(r"\s+").expect("failed to parse regex");
+        let text = re.replace_all(&text, " ").to_string();
+
+        Ok(text)
+    }
+}
+
+struct InlineStyle;
+
+#[async_trait]
+impl Transformer for InlineStyle {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        let css = concat!(
+            "/* chrome-default.css */\n",
+            include_str!("chrome-default.css"),
+            //"\n/* mvp.css */\n",
+            //include_str!("mvp.css"),
+            //"\n/* Xinu Specific overrides */\n",
+            //include_str!("custom.css"),
+        );
+        let inline_opts = InlineOptions {
+            inline_style_tags: true,
+            keep_style_tags: false,
+            keep_link_tags: true,
+            base_url: None,
+            load_remote_stylesheets: true,
+            extra_css: Some(css.into()),
+            preallocate_node_capacity: 32,
+            ..InlineOptions::default()
+        };
+
+        //info!("HTML:\n{html}");
+        Ok(match CSSInliner::new(inline_opts).inline(&html) {
+            Ok(inlined_html) => inlined_html,
+            Err(err) => {
+                error!("failed to inline CSS: {err}");
+                html.to_string()
+            }
+        })
+    }
+}
+
+/// Process images will extract any alt or title tags on images and place them as labels below said
+/// image. It also handles data-src and data-cfsrc attributes
+struct FrameImages;
+
+#[async_trait]
+impl Transformer for FrameImages {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        Ok(rewrite_str(
+            html,
+            RewriteStrSettings {
+                element_content_handlers: vec![
+                    element!("img[data-src]", |el| {
+                        let src = el
+                            .get_attribute("data-src")
+                            .unwrap_or("https://placehold.co/600x400".to_string());
+                        el.set_attribute("src", &src)?;
+
+                        Ok(())
+                    }),
+                    element!("img[data-cfsrc]", |el| {
+                        let src = el
+                            .get_attribute("data-cfsrc")
+                            .unwrap_or("https://placehold.co/600x400".to_string());
+                        el.set_attribute("src", &src)?;
+
+                        Ok(())
+                    }),
+                    element!("img[alt], img[title]", |el| {
+                        let src = el
+                            .get_attribute("src")
+                            .unwrap_or("https://placehold.co/600x400".to_string());
+                        let alt = el.get_attribute("alt");
+                        let title = el.get_attribute("title");
+                        let mut frags =
+                            vec!["<figure>".to_string(), format!(r#"<img src="{src}">"#)];
+                        alt.map(|t| {
+                            if !t.is_empty() {
+                                frags.push(format!("<figcaption>Alt: {t}</figcaption>"))
+                            }
+                        });
+                        title.map(|t| {
+                            if !t.is_empty() {
+                                frags.push(format!("<figcaption>Title: {t}</figcaption>"))
+                            }
+                        });
+                        frags.push("</figure>".to_string());
+                        el.replace(&frags.join("\n"), ContentType::Html);
+
+                        Ok(())
+                    }),
+                ],
+                ..RewriteStrSettings::default()
+            },
+        )?)
+    }
+}
+struct AddOutlink;
+
+#[async_trait]
+impl Transformer for AddOutlink {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        if let Some(link) = link {
+            link.scheme().starts_with("http") && !html.contains(link.as_str())
+        } else {
+            false
+        }
+    }
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        if let Some(link) = link {
+            Ok(format!(
+                r#"
+                {html}
+                <div><a href="{}">View on site</a></div>
+                "#,
+                link
+            ))
+        } else {
+            Ok(html.to_string())
+        }
+    }
+}
+
+struct SlurpContents<'c> {
+    cacher: &'c FilesystemCacher,
+    inline_css: bool,
+    site_selectors: HashMap<String, Vec<Selector>>,
+}
+
+impl<'c> SlurpContents<'c> {
+    fn get_selectors(&self, link: &Url) -> Option<&[Selector]> {
+        for (host, selector) in self.site_selectors.iter() {
+            if link.host_str().map(|h| h.contains(host)).unwrap_or(false) {
+                return Some(&selector);
+            }
+        }
+        None
+    }
+}
+
+#[async_trait]
+impl<'c> Transformer for SlurpContents<'c> {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("SlurpContents should_run {link:?}");
+        let mut will_slurp = false;
+        if let Some(link) = link {
+            will_slurp = self.get_selectors(link).is_some();
+        }
+        if !will_slurp && self.inline_css {
+            return InlineStyle {}.should_run(link, html);
+        }
+        will_slurp
+    }
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("SlurpContents {link:?}");
+        let retryable_status: HashSet<StatusCode> = vec![
+            StatusCode::UNAUTHORIZED,
+            StatusCode::FORBIDDEN,
+            StatusCode::REQUEST_TIMEOUT,
+            StatusCode::TOO_MANY_REQUESTS,
+        ]
+        .into_iter()
+        .collect();
+        if let Some(test_link) = link {
+            // If SlurpContents is configured for inline CSS, but no
+            // configuration found for this site, use the local InlineStyle
+            // transform.
+            if self.inline_css && self.get_selectors(test_link).is_none() {
+                debug!("local inline CSS for {link:?}");
+                return InlineStyle {}.transform(link, html).await;
+            }
+        }
+        let Some(link) = link else {
+            return Ok(html.to_string());
+        };
+        let Some(selectors) = self.get_selectors(&link) else {
+            return Ok(html.to_string());
+        };
+        let cacher = self.cacher;
+        let body = if let Some(body) = cacher.get(link.as_str()) {
+            String::from_utf8_lossy(&body).to_string()
+        } else {
+            let resp = reqwest::get(link.as_str()).await?;
+            let status = resp.status();
+            if status.is_server_error() || retryable_status.contains(&status) {
+                return Err(TransformError::RetryableHttpStatusError(
+                    status,
+                    link.to_string(),
+                ));
+            }
+            if !status.is_success() {
+                return Ok(html.to_string());
+            }
+            let body = resp.text().await?;
+            cacher.set(link.as_str(), body.as_bytes());
+            body
+        };
+        let body = Arc::new(body);
+        let base_url = Some(link.clone());
+        let body = if self.inline_css {
+            debug!("inlining CSS for {link}");
+            let inner_body = Arc::clone(&body);
+            let res = tokio::task::spawn_blocking(move || {
+                let css = concat!(
+                    "/* chrome-default.css */\n",
+                    include_str!("chrome-default.css"),
+                    "\n/* vars.css */\n",
+                    include_str!("../static/vars.css"),
+                    //"\n/* Xinu Specific overrides */\n",
+                    //include_str!("custom.css"),
+                );
+                let res = CSSInliner::options()
+                    .base_url(base_url)
+                    .extra_css(Some(std::borrow::Cow::Borrowed(css)))
+                    .build()
+                    .inline(&inner_body);
+
+                match res {
+                    Ok(inlined_html) => inlined_html,
+                    Err(err) => {
+                        error!("failed to inline remote CSS: {err}");
+                        Arc::into_inner(inner_body).expect("failed to take body out of Arc")
+                    }
+                }
+            })
+            .await;
+            match res {
+                Ok(inlined_html) => inlined_html,
+                Err(err) => {
+                    error!("failed to spawn inline remote CSS: {err}");
+                    Arc::into_inner(body).expect("failed to take body out of Arc")
+                }
+            }
+        } else {
+            debug!("using body as-is for {link:?}");
+            Arc::into_inner(body).expect("failed to take body out of Arc")
+        };
+
+        let doc = Html::parse_document(&body);
+
+        let mut results = Vec::new();
+        for selector in selectors {
+            for frag in doc.select(&selector) {
+                results.push(frag.html())
+                // TODO: figure out how to warn if there were no hits
+                //warn!("couldn't find '{:?}' in {}", selector, link);
+            }
+        }
+        Ok(results.join("<br>"))
+    }
+}
+
+pub fn linkify_html(text: &str) -> String {
+    let mut finder = LinkFinder::new();
+    let finder = finder.url_must_have_scheme(false).kinds(&[LinkKind::Url]);
+    let mut parts = Vec::new();
+    for span in finder.spans(text) {
+        // TODO(wathiede): use Cow<str>?
+        match span.kind() {
+            // Text as-is
+            None => parts.push(span.as_str().to_string()),
+            // Wrap in anchor tag
+            Some(LinkKind::Url) => {
+                let text = span.as_str();
+                let schema = if text.starts_with("http") {
+                    ""
+                } else {
+                    "http://"
+                };
+                let a = format!(r#"<a href="{schema}{0}">{0}</a>"#, text);
+                parts.push(a);
+            }
+            _ => todo!("unhandled kind: {:?}", span.kind().unwrap()),
+        }
+    }
+    parts.join("")
+}
+
+// html contains the content to be cleaned, and cid_prefix is used to resolve mixed part image
+// referrences
+pub fn sanitize_html(
+    html: &str,
+    cid_prefix: &str,
+    base_url: &Option<Url>,
+) -> Result<String, TransformError> {
+    let inline_opts = InlineOptions {
+        inline_style_tags: true,
+        keep_style_tags: true,
+        keep_link_tags: false,
+        base_url: None,
+        load_remote_stylesheets: false,
+        extra_css: None,
+        preallocate_node_capacity: 32,
+        ..InlineOptions::default()
+    };
+
+    let html = match CSSInliner::new(inline_opts).inline(&html) {
+        Ok(inlined_html) => inlined_html,
+        Err(err) => {
+            error!("failed to inline CSS: {err}");
+            html.to_string()
+        }
+    };
+    let mut element_content_handlers = vec![
+        // Open links in new tab
+        element!("a[href]", |el| {
+            el.set_attribute("target", "_blank").unwrap();
+
+            Ok(())
+        }),
+        // Replace mixed part CID images with URL
+        element!("img[src]", |el| {
+            let src = el
+                .get_attribute("src")
+                .expect("src was required")
+                .replace("cid:", cid_prefix);
+
+            el.set_attribute("src", &src)?;
+
+            Ok(())
+        }),
+        // Only secure image URLs
+        element!("img[src]", |el| {
+            let src = el
+                .get_attribute("src")
+                .expect("src was required")
+                .replace("http:", "https:");
+
+            el.set_attribute("src", &src)?;
+
+            Ok(())
+        }),
+        // Add https to href with //<domain name>
+        element!("link[href]", |el| {
+            info!("found link[href] {el:?}");
+            let mut href = el.get_attribute("href").expect("href was required");
+            if href.starts_with("//") {
+                warn!("adding https to {href}");
+                href.insert_str(0, "https:");
+            }
+
+            el.set_attribute("href", &href)?;
+
+            Ok(())
+        }),
+        // Add https to src with //<domain name>
+        element!("style[src]", |el| {
+            let mut src = el.get_attribute("src").expect("src was required");
+            if src.starts_with("//") {
+                src.insert_str(0, "https:");
+            }
+
+            el.set_attribute("src", &src)?;
+
+            Ok(())
+        }),
+    ];
+    if let Some(base_url) = base_url {
+        element_content_handlers.extend(vec![
+            // Make links with relative URLs absolute
+            element!("a[href]", |el| {
+                if let Some(Ok(href)) = el.get_attribute("href").map(|href| base_url.join(&href)) {
+                    el.set_attribute("href", &href.as_str()).unwrap();
+                }
+
+                Ok(())
+            }),
+            // Make images with relative srcs absolute
+            element!("img[src]", |el| {
+                if let Some(Ok(src)) = el.get_attribute("src").map(|src| base_url.join(&src)) {
+                    el.set_attribute("src", &src.as_str()).unwrap();
+                }
+
+                Ok(())
+            }),
+        ]);
+    }
+    let html = rewrite_str(
+        &html,
+        RewriteStrSettings {
+            element_content_handlers,
+            ..RewriteStrSettings::default()
+        },
+    )?;
+    // Default's don't allow style, but we want to preserve that.
+    // TODO: remove 'class' if rendering mails moves to a two phase process where abstract message
+    // types are collected, santized, and then grouped together as one big HTML doc
+    let attributes = hashset![
+        "align", "bgcolor", "class", "color", "height", "lang", "title", "width", "style",
+    ];
+
+    let tags = hashset![
+        "a",
+        "abbr",
+        "acronym",
+        "area",
+        "article",
+        "aside",
+        "b",
+        "bdi",
+        "bdo",
+        "blockquote",
+        "br",
+        "caption",
+        "center",
+        "cite",
+        "code",
+        "col",
+        "colgroup",
+        "data",
+        "dd",
+        "del",
+        "details",
+        "dfn",
+        "div",
+        "dl",
+        "dt",
+        "em",
+        "figcaption",
+        "figure",
+        "footer",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "header",
+        "hgroup",
+        "hr",
+        "i",
+        "iframe", // wathiede
+        "img",
+        "ins",
+        "kbd",
+        "kbd",
+        "li",
+        "map",
+        "mark",
+        "nav",
+        "noscript", // wathiede
+        "ol",
+        "p",
+        "pre",
+        "q",
+        "rp",
+        "rt",
+        "rtc",
+        "ruby",
+        "s",
+        "samp",
+        "small",
+        "span",
+        "strike",
+        "strong",
+        "sub",
+        "summary",
+        "sup",
+        "table",
+        "tbody",
+        "td",
+        "th",
+        "thead",
+        "time",
+        "title", // wathiede
+        "tr",
+        "tt",
+        "u",
+        "ul",
+        "var",
+        "wbr",
+    ];
+    let tag_attributes = hashmap![
+        "a" => hashset![
+            "href", "hreflang", "target",
+        ],
+        "bdo" => hashset![
+            "dir"
+        ],
+        "blockquote" => hashset![
+            "cite"
+        ],
+        "col" => hashset![
+            "align", "char", "charoff", "span"
+        ],
+        "colgroup" => hashset![
+            "align", "char", "charoff", "span"
+        ],
+        "del" => hashset![
+            "cite", "datetime"
+        ],
+        "hr" => hashset![
+            "align", "size", "width"
+        ],
+        "iframe" => hashset![
+            "src", "allow", "allowfullscreen"
+        ],
+        "img" => hashset![
+            "align", "alt", "height", "src", "width"
+        ],
+        "ins" => hashset![
+            "cite", "datetime"
+        ],
+        "ol" => hashset![
+            "start"
+        ],
+        "q" => hashset![
+            "cite"
+        ],
+        "table" => hashset![
+            "align", "border", "cellpadding", "cellspacing", "char", "charoff", "summary",
+        ],
+        "tbody" => hashset![
+            "align", "char", "charoff"
+        ],
+        "td" => hashset![
+            "align", "char", "charoff", "colspan", "headers", "rowspan"
+        ],
+        "tfoot" => hashset![
+            "align", "char", "charoff"
+        ],
+        "th" => hashset![
+            "align", "char", "charoff", "colspan", "headers", "rowspan", "scope"
+        ],
+        "thead" => hashset![
+            "align", "char", "charoff"
+        ],
+        "tr" => hashset![
+            "align", "char", "charoff"
+        ],
+    ];
+
+    let html = ammonia::Builder::default()
+        .tags(tags)
+        .tag_attributes(tag_attributes)
+        .generic_attributes(attributes)
+        .clean(&html)
+        .to_string();
+
+    Ok(html)
+}
+
+fn compute_offset_limit(
+    after: Option<i32>,
+    before: Option<i32>,
+    first: Option<i32>,
+    last: Option<i32>,
+) -> (i32, i32) {
+    let default_page_size = 10000;
+    match (after, before, first, last) {
+        // Reasonable defaults
+        (None, None, None, None) => (0, default_page_size),
+        (None, None, Some(first), None) => (0, first),
+        (Some(after), None, None, None) => (after + 1, default_page_size),
+        (Some(after), None, Some(first), None) => (after + 1, first),
+        (None, Some(before), None, None) => (0.max(before - default_page_size), default_page_size),
+        (None, Some(before), None, Some(last)) => (0.max(before - last), last),
+        (None, None, None, Some(_)) => {
+            panic!("specifying last and no before doesn't make sense")
+        }
+        (None, None, Some(_), Some(_)) => {
+            panic!("specifying first and last doesn't make sense")
+        }
+        (None, Some(_), Some(_), _) => {
+            panic!("specifying before and first doesn't make sense")
+        }
+        (Some(_), Some(_), _, _) => {
+            panic!("specifying after and before doesn't make sense")
+        }
+        (Some(_), None, None, Some(_)) => {
+            panic!("specifying after and last doesn't make sense")
+        }
+        (Some(_), None, Some(_), Some(_)) => {
+            panic!("specifying after, first and last doesn't make sense")
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct Query {
+    pub unread_only: bool,
+    pub tags: Vec<String>,
+    pub uids: Vec<String>,
+    pub remainder: Vec<String>,
+    pub is_notmuch: bool,
+    pub is_newsreader: bool,
+    pub is_tantivy: bool,
+    pub corpus: Option<Corpus>,
+}
+
+impl fmt::Display for Query {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+        if self.unread_only {
+            write!(f, "is:unread ")?;
+        }
+        for tag in &self.tags {
+            write!(f, "tag:{tag} ")?;
+        }
+        for uid in &self.uids {
+            write!(f, "id:{uid} ")?;
+        }
+        if self.is_notmuch {
+            write!(f, "is:mail ")?;
+        }
+        if self.is_newsreader {
+            write!(f, "is:newsreader ")?;
+        }
+        if self.is_newsreader {
+            write!(f, "is:news ")?;
+        }
+        match self.corpus {
+            Some(c) => write!(f, "corpus:{c:?}")?,
+            _ => (),
+        }
+        for rem in &self.remainder {
+            write!(f, "{rem} ")?;
+        }
+        Ok(())
+    }
+}
+
+impl Query {
+    // Converts the internal state of Query to something suitable for notmuch queries. Removes and
+    // letterbox specific '<key>:<value' tags
+    fn to_notmuch(&self) -> String {
+        let mut parts = Vec::new();
+        if !self.is_notmuch {
+            return String::new();
+        }
+
+        if self.unread_only {
+            parts.push("is:unread".to_string());
+        }
+        for tag in &self.tags {
+            parts.push(format!("tag:{tag}"));
+        }
+        for uid in &self.uids {
+            parts.push(uid.clone());
+        }
+        for r in &self.remainder {
+            // Rewrite "to:" to include ExtraTo:. ExtraTo: is configured in
+            // notmuch-config to index Delivered-To and X-Original-To headers.
+            if r.starts_with("to:") {
+                parts.push("(".to_string());
+                parts.push(r.to_string());
+                parts.push("OR".to_string());
+                parts.push(r.replace("to:", "ExtraTo:"));
+                parts.push(")".to_string());
+            } else {
+                parts.push(r.to_string());
+            }
+        }
+        parts.join(" ")
+    }
+}
+
+impl FromStr for Query {
+    type Err = Infallible;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut unread_only = false;
+        let mut tags = Vec::new();
+        let mut uids = Vec::new();
+        let mut remainder = Vec::new();
+        let mut is_notmuch = false;
+        let mut is_newsreader = false;
+        let mut is_tantivy = false;
+        let mut corpus = None;
+        for word in s.split_whitespace() {
+            if word == "is:unread" {
+                unread_only = true
+            } else if word.starts_with("tag:") {
+                let t = &word["tag:".len()..];
+                // Per-address emails are faked as `tag:@<domain>/<username>`, rewrite to `to:` form
+                if t.starts_with('@') && t.contains('.') {
+                    let t = match t.split_once('/') {
+                        None => format!("to:{t}"),
+                        Some((domain, user)) => format!("to:{user}{domain}"),
+                    };
+                    remainder.push(t);
+                } else {
+                    tags.push(t.to_string());
+                };
+
+                /*
+                            } else if word.starts_with("tag:") {
+                                // Any tag that doesn't match site_prefix should explicitly set the site to something not in the
+                                // database
+                                site = Some(NON_EXISTENT_SITE_NAME.to_string());
+                */
+            } else if word.starts_with("corpus:") {
+                let c = word["corpus:".len()..].to_string();
+                corpus = c.parse::<Corpus>().map(|c| Some(c)).unwrap_or_else(|e| {
+                    warn!("Error parsing corpus '{c}': {e:?}");
+                    None
+                });
+            } else if is_newsreader_thread(word) {
+                uids.push(word.to_string());
+            } else if is_notmuch_thread_or_id(word) {
+                uids.push(word.to_string());
+            } else if word == "is:mail" || word == "is:email" || word == "is:notmuch" {
+                is_notmuch = true;
+            } else if word == "is:news" {
+                is_newsreader = true;
+            } else if word == "is:newsreader" {
+                is_newsreader = true;
+            } else {
+                remainder.push(word.to_string());
+            }
+        }
+        // If we don't see any explicit filters for a corpus, flip them all on
+        if corpus.is_none() && !(is_notmuch || is_tantivy || is_newsreader) {
+            is_notmuch = true;
+            is_newsreader = true;
+            is_tantivy = true;
+        }
+        Ok(Query {
+            unread_only,
+            tags,
+            uids,
+            remainder,
+            is_notmuch,
+            is_newsreader,
+            is_tantivy,
+            corpus,
+        })
+    }
+}
+pub struct ThreadSummaryRecord {
+    pub site: Option<String>,
+    pub date: Option<PrimitiveDateTime>,
+    pub is_read: Option<bool>,
+    pub title: Option<String>,
+    pub uid: String,
+    pub name: Option<String>,
+    pub corpus: Corpus,
+}
+
+async fn thread_summary_from_row(r: ThreadSummaryRecord) -> ThreadSummary {
+    let site = r.site.unwrap_or("UNKOWN TAG".to_string());
+    let mut tags = vec![format!("{NEWSREADER_TAG_PREFIX}{site}")];
+    if !r.is_read.unwrap_or(true) {
+        tags.push("unread".to_string());
+    };
+    let mut title = r.title.unwrap_or("NO TITLE".to_string());
+    title = clean_title(&title).await.expect("failed to clean title");
+    ThreadSummary {
+        thread: format!("{NEWSREADER_THREAD_PREFIX}{}", r.uid),
+        timestamp: r
+            .date
+            .expect("post missing date")
+            .assume_utc()
+            .unix_timestamp() as isize,
+        date_relative: format!("{:?}", r.date),
+        //date_relative: "TODO date_relative".to_string(),
+        matched: 0,
+        total: 1,
+        authors: r.name.unwrap_or_else(|| site.clone()),
+        subject: title,
+        tags,
+        corpus: r.corpus,
+    }
+}
+async fn clean_title(title: &str) -> Result<String, ServerError> {
+    // Make title HTML so html parsers work
+    let mut title = format!("<html>{title}</html>");
+    let title_tranformers: Vec<Box<dyn Transformer>> =
+        vec![Box::new(EscapeHtml), Box::new(StripHtml)];
+    // Make title HTML so html parsers work
+    title = format!("<html>{title}</html>");
+    for t in title_tranformers.iter() {
+        if t.should_run(&None, &title) {
+            title = t.transform(&None, &title).await?;
+        }
+    }
+    Ok(title)
+}

server/src/mail.rs

@@ -0,0 +1,113 @@
+use std::{fs::File, io::Read};
+
+use mailparse::{
+    addrparse_header, dateparse, parse_mail, MailHeaderMap, MailParseError, ParsedMail,
+};
+use sqlx::postgres::PgPool;
+use thiserror::Error;
+use tracing::info;
+
+#[derive(Error, Debug)]
+pub enum MailError {
+    #[error("missing from header")]
+    MissingFrom,
+    #[error("missing from header display name")]
+    MissingFromDisplayName,
+    #[error("missing subject header")]
+    MissingSubject,
+    #[error("missing html part")]
+    MissingHtmlPart,
+    #[error("missing message ID")]
+    MissingMessageId,
+    #[error("missing date")]
+    MissingDate,
+    #[error("DB error {0}")]
+    SqlxError(#[from] sqlx::Error),
+    #[error("IO error {0}")]
+    IOError(#[from] std::io::Error),
+    #[error("mail parse error {0}")]
+    MailParseError(#[from] MailParseError),
+}
+
+pub async fn read_mail_to_db(pool: &PgPool, path: &str) -> Result<(), MailError> {
+    let mut file = File::open(path)?;
+    let mut buffer = Vec::new();
+    file.read_to_end(&mut buffer)?;
+    let m = parse_mail(&buffer)?;
+
+    let subject = m
+        .headers
+        .get_first_value("subject")
+        .ok_or(MailError::MissingSubject)?;
+
+    let from = addrparse_header(
+        m.headers
+            .get_first_header("from")
+            .ok_or(MailError::MissingFrom)?,
+    )?;
+    let from = from.extract_single_info().ok_or(MailError::MissingFrom)?;
+    let name = from.display_name.ok_or(MailError::MissingFromDisplayName)?;
+    let slug = name.to_lowercase().replace(' ', "-");
+    let url = from.addr;
+    let message_id = m
+        .headers
+        .get_first_value("Message-ID")
+        .ok_or(MailError::MissingMessageId)?;
+    let uid = &message_id;
+    let feed_id = find_feed(&pool, &name, &slug, &url).await?;
+    let date = dateparse(
+        &m.headers
+            .get_first_value("Date")
+            .ok_or(MailError::MissingDate)?,
+    )?;
+
+    println!("Feed: {feed_id} Subject: {}", subject);
+
+    if let Some(_m) = first_html(&m) {
+        info!("add email {slug} {subject} {message_id} {date} {uid} {url}");
+    } else {
+        return Err(MailError::MissingHtmlPart.into());
+    }
+
+    Ok(())
+}
+fn first_html<'m>(m: &'m ParsedMail<'m>) -> Option<&'m ParsedMail<'m>> {
+    for ele in m.parts() {
+        if ele.ctype.mimetype == "text/html" {
+            return Some(ele);
+        }
+    }
+    None
+}
+async fn find_feed(pool: &PgPool, name: &str, slug: &str, url: &str) -> Result<i32, MailError> {
+    match sqlx::query!(
+        r#"
+SELECT id
+FROM feed
+WHERE slug = $1
+        "#,
+        slug
+    )
+    .fetch_one(pool)
+    .await
+    {
+        Err(sqlx::Error::RowNotFound) => {
+            let rec = sqlx::query!(
+                r#"
+INSERT INTO feed ( name, slug, url, homepage, selector )
+VALUES ( $1, $2, $3, '', '' )
+RETURNING id
+        "#,
+                name,
+                slug,
+                url
+            )
+            .fetch_one(pool)
+            .await?;
+
+            return Ok(rec.id);
+        }
+        Ok(rec) => return Ok(rec.id),
+        Err(e) => return Err(e.into()),
+    };
+}

server/src/mvp.css

@@ -0,0 +1,498 @@
+/* MVP.css v1.15 - https://github.com/andybrewer/mvp */
+
+/* :root content stored in client side index.html */
+
+html {
+  scroll-behavior: smooth;
+}
+
+@media (prefers-reduced-motion: reduce) {
+  html {
+    scroll-behavior: auto;
+  }
+}
+
+/* Layout */
+article aside {
+  background: var(--color-secondary-accent);
+  border-left: 4px solid var(--color-secondary);
+  padding: 0.01rem 0.8rem;
+}
+
+body {
+  background: var(--color-bg);
+  color: var(--color-text);
+  font-family: var(--font-family);
+  line-height: var(--line-height);
+  margin: 0;
+  overflow-x: hidden;
+  padding: 0;
+}
+
+footer,
+header,
+main {
+  margin: 0 auto;
+  max-width: var(--width-content);
+  padding: 3rem 1rem;
+}
+
+hr {
+  background-color: var(--color-bg-secondary);
+  border: none;
+  height: 1px;
+  margin: 4rem 0;
+  width: 100%;
+}
+
+section {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: var(--justify-important);
+}
+
+section img,
+article img {
+  max-width: 100%;
+}
+
+section pre {
+  overflow: auto;
+}
+
+section aside {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  margin: 1rem;
+  padding: 1.25rem;
+  width: var(--width-card);
+}
+
+section aside:hover {
+  box-shadow: var(--box-shadow) var(--color-bg-secondary);
+}
+
+[hidden] {
+  display: none;
+}
+
+/* Headers */
+article header,
+div header,
+main header {
+  padding-top: 0;
+}
+
+header {
+  text-align: var(--justify-important);
+}
+
+header a b,
+header a em,
+header a i,
+header a strong {
+  margin-left: 0.5rem;
+  margin-right: 0.5rem;
+}
+
+header nav img {
+  margin: 1rem 0;
+}
+
+section header {
+  padding-top: 0;
+  width: 100%;
+}
+
+/* Nav */
+nav {
+  align-items: center;
+  display: flex;
+  font-weight: bold;
+  justify-content: space-between;
+  margin-bottom: 7rem;
+}
+
+nav ul {
+  list-style: none;
+  padding: 0;
+}
+
+nav ul li {
+  display: inline-block;
+  margin: 0 0.5rem;
+  position: relative;
+  text-align: left;
+}
+
+/* Nav Dropdown */
+nav ul li:hover ul {
+  display: block;
+}
+
+nav ul li ul {
+  background: var(--color-bg);
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  display: none;
+  height: auto;
+  left: -2px;
+  padding: .5rem 1rem;
+  position: absolute;
+  top: 1.7rem;
+  white-space: nowrap;
+  width: auto;
+  z-index: 1;
+}
+
+nav ul li ul::before {
+  /* fill gap above to make mousing over them easier */
+  content: "";
+  position: absolute;
+  left: 0;
+  right: 0;
+  top: -0.5rem;
+  height: 0.5rem;
+}
+
+nav ul li ul li,
+nav ul li ul li a {
+  display: block;
+}
+
+/* Typography */
+code,
+samp {
+  background-color: var(--color-accent);
+  border-radius: var(--border-radius);
+  color: var(--color-text);
+  display: inline-block;
+  margin: 0 0.1rem;
+  padding: 0 0.5rem;
+}
+
+details {
+  margin: 1.3rem 0;
+}
+
+details summary {
+  font-weight: bold;
+  cursor: pointer;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  line-height: var(--line-height);
+  text-wrap: balance;
+}
+
+mark {
+  padding: 0.1rem;
+}
+
+ol li,
+ul li {
+  padding: 0.2rem 0;
+}
+
+p {
+  margin: 0.75rem 0;
+  padding: 0;
+  width: 100%;
+}
+
+pre {
+  margin: 1rem 0;
+  max-width: var(--width-card-wide);
+  padding: 1rem 0;
+}
+
+pre code,
+pre samp {
+  display: block;
+  max-width: var(--width-card-wide);
+  padding: 0.5rem 2rem;
+  white-space: pre-wrap;
+}
+
+small {
+  color: var(--color-text-secondary);
+}
+
+sup {
+  background-color: var(--color-secondary);
+  border-radius: var(--border-radius);
+  color: var(--color-bg);
+  font-size: xx-small;
+  font-weight: bold;
+  margin: 0.2rem;
+  padding: 0.2rem 0.3rem;
+  position: relative;
+  top: -2px;
+}
+
+/* Links */
+a {
+  color: var(--color-link);
+  display: inline-block;
+  font-weight: bold;
+  text-decoration: underline;
+}
+
+a:hover {
+  filter: brightness(var(--hover-brightness));
+}
+
+a:active {
+  filter: brightness(var(--active-brightness));
+}
+
+a b,
+a em,
+a i,
+a strong,
+button,
+input[type="submit"] {
+  border-radius: var(--border-radius);
+  display: inline-block;
+  font-size: medium;
+  font-weight: bold;
+  line-height: var(--line-height);
+  margin: 0.5rem 0;
+  padding: 1rem 2rem;
+}
+
+button,
+input[type="submit"] {
+  font-family: var(--font-family);
+}
+
+button:hover,
+input[type="submit"]:hover {
+  cursor: pointer;
+  filter: brightness(var(--hover-brightness));
+}
+
+button:active,
+input[type="submit"]:active {
+  filter: brightness(var(--active-brightness));
+}
+
+a b,
+a strong,
+button,
+input[type="submit"] {
+  background-color: var(--color-link);
+  border: 2px solid var(--color-link);
+  color: var(--color-bg);
+}
+
+a em,
+a i {
+  border: 2px solid var(--color-link);
+  border-radius: var(--border-radius);
+  color: var(--color-link);
+  display: inline-block;
+  padding: 1rem 2rem;
+}
+
+article aside a {
+  color: var(--color-secondary);
+}
+
+/* Images */
+figure {
+  margin: 0;
+  padding: 0;
+}
+
+figure img {
+  max-width: 100%;
+}
+
+figure figcaption {
+  color: var(--color-text-secondary);
+}
+
+/* Forms */
+button:disabled,
+input:disabled {
+  background: var(--color-bg-secondary);
+  border-color: var(--color-bg-secondary);
+  color: var(--color-text-secondary);
+  cursor: not-allowed;
+}
+
+button[disabled]:hover,
+input[type="submit"][disabled]:hover {
+  filter: none;
+}
+
+form {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  display: block;
+  max-width: var(--width-card-wide);
+  min-width: var(--width-card);
+  padding: 1.5rem;
+  text-align: var(--justify-normal);
+}
+
+form header {
+  margin: 1.5rem 0;
+  padding: 1.5rem 0;
+}
+
+input,
+label,
+select,
+textarea {
+  display: block;
+  font-size: inherit;
+  max-width: var(--width-card-wide);
+}
+
+input[type="checkbox"],
+input[type="radio"] {
+  display: inline-block;
+}
+
+input[type="checkbox"]+label,
+input[type="radio"]+label {
+  display: inline-block;
+  font-weight: normal;
+  position: relative;
+  top: 1px;
+}
+
+input[type="range"] {
+  padding: 0.4rem 0;
+}
+
+input,
+select,
+textarea {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  margin-bottom: 1rem;
+  padding: 0.4rem 0.8rem;
+}
+
+input[type="text"],
+input[type="password"] textarea {
+  width: calc(100% - 1.6rem);
+}
+
+input[readonly],
+textarea[readonly] {
+  background-color: var(--color-bg-secondary);
+}
+
+label {
+  font-weight: bold;
+  margin-bottom: 0.2rem;
+}
+
+/* Popups */
+dialog {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  position: fixed;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  width: 50%;
+  z-index: 999;
+}
+
+/* Tables */
+table {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  border-spacing: 0;
+  display: inline-block;
+  max-width: 100%;
+  overflow-x: auto;
+  padding: 0;
+  white-space: nowrap;
+}
+
+table td,
+table th,
+table tr {
+  padding: 0.4rem 0.8rem;
+  text-align: var(--justify-important);
+}
+
+table thead {
+  background-color: var(--color-table);
+  border-collapse: collapse;
+  border-radius: var(--border-radius);
+  color: var(--color-bg);
+  margin: 0;
+  padding: 0;
+}
+
+table thead tr:first-child th:first-child {
+  border-top-left-radius: var(--border-radius);
+}
+
+table thead tr:first-child th:last-child {
+  border-top-right-radius: var(--border-radius);
+}
+
+table thead th:first-child,
+table tr td:first-child {
+  text-align: var(--justify-normal);
+}
+
+table tr:nth-child(even) {
+  background-color: var(--color-accent);
+}
+
+/* Quotes */
+blockquote {
+  display: block;
+  font-size: x-large;
+  line-height: var(--line-height);
+  margin: 1rem auto;
+  max-width: var(--width-card-medium);
+  padding: 1.5rem 1rem;
+  text-align: var(--justify-important);
+}
+
+blockquote footer {
+  color: var(--color-text-secondary);
+  display: block;
+  font-size: small;
+  line-height: var(--line-height);
+  padding: 1.5rem 0;
+}
+
+/* Scrollbars */
+* {
+  scrollbar-width: thin;
+  scrollbar-color: var(--color-scrollbar) transparent;
+}
+
+*::-webkit-scrollbar {
+  width: 5px;
+  height: 5px;
+}
+
+*::-webkit-scrollbar-track {
+  background: transparent;
+}
+
+*::-webkit-scrollbar-thumb {
+  background-color: var(--color-scrollbar);
+  border-radius: 10px;
+}

server/src/newsreader.rs

@@ -0,0 +1,384 @@
+use std::collections::HashMap;
+
+use cacher::FilesystemCacher;
+use futures::{stream::FuturesUnordered, StreamExt};
+use letterbox_shared::compute_color;
+use log::{error, info};
+use maplit::hashmap;
+use scraper::Selector;
+use sqlx::postgres::PgPool;
+use tracing::instrument;
+use url::Url;
+
+use crate::{
+    clean_title, compute_offset_limit,
+    error::ServerError,
+    graphql::{Corpus, NewsPost, Tag, Thread, ThreadSummary},
+    thread_summary_from_row, AddOutlink, FrameImages, Query, SanitizeHtml, SlurpContents,
+    StripHtml, ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX, NEWSREADER_THREAD_PREFIX,
+};
+
+pub fn is_newsreader_query(query: &Query) -> bool {
+    query.is_newsreader || query.corpus == Some(Corpus::Newsreader)
+}
+
+pub fn is_newsreader_thread(query: &str) -> bool {
+    query.starts_with(NEWSREADER_THREAD_PREFIX)
+}
+
+pub fn extract_thread_id(query: &str) -> &str {
+    if query.starts_with(NEWSREADER_THREAD_PREFIX) {
+        &query[NEWSREADER_THREAD_PREFIX.len()..]
+    } else {
+        query
+    }
+}
+
+pub fn extract_site(tag: &str) -> &str {
+    &tag[NEWSREADER_TAG_PREFIX.len()..]
+}
+
+pub fn make_news_tag(tag: &str) -> String {
+    format!("tag:{NEWSREADER_TAG_PREFIX}{tag}")
+}
+
+fn site_from_tags(tags: &[String]) -> Option<String> {
+    for t in tags {
+        if t.starts_with(NEWSREADER_TAG_PREFIX) {
+            return Some(extract_site(t).to_string());
+        }
+    }
+    None
+}
+
+#[instrument(name = "newsreader::count", skip_all, fields(query=%query))]
+pub async fn count(pool: &PgPool, query: &Query) -> Result<usize, ServerError> {
+    if !is_newsreader_query(query) {
+        return Ok(0);
+    }
+    let site = site_from_tags(&query.tags);
+    if !query.tags.is_empty() && site.is_none() {
+        // Newsreader can only handle all sites read/unread queries, anything with a non-site tag
+        // isn't supported
+        return Ok(0);
+    }
+
+    let search_term = query.remainder.join(" ");
+    let search_term = search_term.trim();
+    let search_term = if search_term.is_empty() {
+        None
+    } else {
+        Some(search_term)
+    };
+    // TODO: add support for looking for search_term in title and site
+    let row = sqlx::query_file!("sql/count.sql", site, query.unread_only, search_term)
+        .fetch_one(pool)
+        .await?;
+    Ok(row.count.unwrap_or(0).try_into().unwrap_or(0))
+}
+
+#[instrument(name = "newsreader::search", skip_all, fields(query=%query))]
+pub async fn search(
+    pool: &PgPool,
+    after: Option<i32>,
+    before: Option<i32>,
+    first: Option<i32>,
+    last: Option<i32>,
+    query: &Query,
+) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
+    info!("search({after:?} {before:?} {first:?} {last:?} {query:?}");
+    if !is_newsreader_query(query) {
+        return Ok(Vec::new());
+    }
+    let site = site_from_tags(&query.tags);
+    if !query.tags.is_empty() && site.is_none() {
+        // Newsreader can only handle all sites read/unread queries, anything with a non-site tag
+        // isn't supported
+        return Ok(Vec::new());
+    }
+
+    let (offset, mut limit) = compute_offset_limit(after, before, first, last);
+    if before.is_none() {
+        // When searching forward, the +1 is to see if there are more pages of data available.
+        // Searching backwards implies there's more pages forward, because the value represented by
+        // `before` is on the next page.
+        limit = limit + 1;
+    }
+
+    info!(
+        "search offset {offset} limit {limit} site {site:?} unread_only {}",
+        query.unread_only
+    );
+    let search_term = query.remainder.join(" ");
+    let search_term = search_term.trim();
+    let search_term = if search_term.is_empty() {
+        None
+    } else {
+        Some(search_term)
+    };
+
+    // TODO: add support for looking for search_term in title and site
+    let rows = sqlx::query_file!(
+        "sql/threads.sql",
+        site,
+        query.unread_only,
+        offset as i64,
+        limit as i64,
+        search_term
+    )
+    .fetch_all(pool)
+    .await?;
+    let mut res = Vec::new();
+    for (i, r) in rows.into_iter().enumerate() {
+        res.push((
+            i as i32 + offset,
+            thread_summary_from_row(ThreadSummaryRecord {
+                site: r.site,
+                date: r.date,
+                is_read: r.is_read,
+                title: r.title,
+                uid: r.uid,
+                name: r.name,
+                corpus: Corpus::Newsreader,
+            })
+            .await,
+        ));
+    }
+    Ok(res)
+}
+#[instrument(name = "newsreader::tags", skip_all, fields(needs_unread=%_needs_unread))]
+pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
+    // TODO: optimize query by using needs_unread
+    let tags = sqlx::query_file!("sql/tags.sql").fetch_all(pool).await?;
+    let tags = tags
+        .into_iter()
+        .map(|tag| {
+            let unread = tag.unread.unwrap_or(0).try_into().unwrap_or(0);
+            let name = format!(
+                "{NEWSREADER_TAG_PREFIX}{}",
+                tag.site.expect("tag must have site")
+            );
+            let hex = compute_color(&name);
+            Tag {
+                name,
+                fg_color: "white".to_string(),
+                bg_color: hex,
+                unread,
+            }
+        })
+        .collect();
+    Ok(tags)
+}
+
+#[instrument(name = "newsreader::thread", skip_all, fields(thread_id=%thread_id))]
+pub async fn thread(
+    cacher: &FilesystemCacher,
+    pool: &PgPool,
+    thread_id: String,
+) -> Result<Thread, ServerError> {
+    let id = thread_id
+        .strip_prefix(NEWSREADER_THREAD_PREFIX)
+        .expect("news thread doesn't start with '{NEWSREADER_THREAD_PREFIX}'")
+        .to_string();
+
+    let r = sqlx::query_file!("sql/thread.sql", id)
+        .fetch_one(pool)
+        .await?;
+
+    let slug = r.site.unwrap_or("no-slug".to_string());
+    let site = r.name.unwrap_or("NO SITE".to_string());
+    // TODO: remove the various places that have this as an Option
+    let link = Some(Url::parse(&r.link)?);
+    let mut body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
+    let body_transformers: Vec<Box<dyn Transformer>> = vec![
+        Box::new(SlurpContents {
+            cacher,
+            inline_css: true,
+            site_selectors: slurp_contents_selectors(),
+        }),
+        Box::new(FrameImages),
+        Box::new(AddOutlink),
+        // TODO: causes doubling of images in cloudflare blogs
+        //Box::new(EscapeHtml),
+        Box::new(SanitizeHtml {
+            cid_prefix: "",
+            base_url: &link,
+        }),
+    ];
+    for t in body_transformers.iter() {
+        if t.should_run(&link, &body) {
+            body = t.transform(&link, &body).await?;
+        }
+    }
+    let title = clean_title(&r.title.unwrap_or("NO TITLE".to_string())).await?;
+    let is_read = r.is_read.unwrap_or(false);
+    let timestamp = r
+        .date
+        .expect("post missing date")
+        .assume_utc()
+        .unix_timestamp();
+    Ok(Thread::News(NewsPost {
+        thread_id,
+        is_read,
+        slug,
+        site,
+        title,
+        body,
+        url: link
+            .as_ref()
+            .map(|url| url.to_string())
+            .unwrap_or("NO URL".to_string()),
+        timestamp,
+    }))
+}
+#[instrument(name = "newsreader::set_read_status", skip_all, fields(query=%query,unread=%unread))]
+pub async fn set_read_status<'ctx>(
+    pool: &PgPool,
+    query: &Query,
+    unread: bool,
+) -> Result<bool, ServerError> {
+    // TODO: make single query when query.uids.len() > 1
+    let uids: Vec<_> = query
+        .uids
+        .iter()
+        .filter(|uid| is_newsreader_thread(uid))
+        .map(
+            |uid| extract_thread_id(uid), // TODO strip prefix
+        )
+        .collect();
+    for uid in uids {
+        sqlx::query_file!("sql/set_unread.sql", !unread, uid)
+            .execute(pool)
+            .await?;
+    }
+    Ok(true)
+}
+#[instrument(name = "newsreader::refresh", skip_all)]
+pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<bool, ServerError> {
+    async fn update_search_summary(
+        pool: &PgPool,
+        cacher: &FilesystemCacher,
+        link: String,
+        body: String,
+        id: i32,
+    ) -> Result<(), ServerError> {
+        let slurp_contents = SlurpContents {
+            cacher,
+            inline_css: true,
+            site_selectors: slurp_contents_selectors(),
+        };
+        let strip_html = StripHtml;
+
+        info!("adding {link} to search index");
+        let mut body = body;
+        if let Ok(link) = Url::parse(&link) {
+            let link = Some(link);
+            if slurp_contents.should_run(&link, &body) {
+                body = slurp_contents.transform(&link, &body).await?;
+            }
+        } else {
+            error!("failed to parse link: {}", link);
+        }
+        body = strip_html.transform(&None, &body).await?;
+        sqlx::query!(
+            "UPDATE post SET search_summary = $1 WHERE id = $2",
+            body,
+            id
+        )
+        .execute(pool)
+        .await?;
+        Ok(())
+    }
+
+    let mut unordered: FuturesUnordered<_> = sqlx::query_file!("sql/need-search-summary.sql",)
+        .fetch_all(pool)
+        .await?
+        .into_iter()
+        .filter_map(|r| {
+            let Some(body) = r.clean_summary else {
+                error!("clean_summary missing for {}", r.link);
+                return None;
+            };
+            let id = r.id;
+            Some(update_search_summary(pool, cacher, r.link, body, id))
+        })
+        .collect();
+
+    while let Some(res) = unordered.next().await {
+        //let res = res;
+        match res {
+            Ok(()) => {}
+            Err(err) => {
+                info!("failed refresh {err:?}");
+                // TODO:
+                //fd.error = Some(err);
+            }
+        };
+    }
+    Ok(true)
+}
+
+fn slurp_contents_selectors() -> HashMap<String, Vec<Selector>> {
+    hashmap![
+        "atmeta.com".to_string() => vec![
+            Selector::parse("div.entry-content").unwrap(),
+        ],
+        "blog.prusa3d.com".to_string() => vec![
+            Selector::parse("article.content .post-block").unwrap(),
+        ],
+        "blog.cloudflare.com".to_string() => vec![
+            Selector::parse(".author-lists .author-name-tooltip").unwrap(),
+            Selector::parse(".post-full-content").unwrap()
+        ],
+        "blog.zsa.io".to_string() => vec![
+            Selector::parse("section.blog-article").unwrap(),
+        ],
+        "engineering.fb.com".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "grafana.com".to_string() => vec![
+            Selector::parse(".blog-content").unwrap(),
+        ],
+        "hackaday.com".to_string() => vec![
+            Selector::parse("div.entry-featured-image").unwrap(),
+            Selector::parse("div.entry-content").unwrap()
+        ],
+        "ingowald.blog".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "jvns.ca".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
+        "natwelch.com".to_string() => vec![
+            Selector::parse("article div.prose").unwrap(),
+        ],
+        "rustacean-station.org".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "slashdot.org".to_string() => vec![
+            Selector::parse("span.story-byline").unwrap(),
+            Selector::parse("div.p").unwrap(),
+        ],
+        "theonion.com".to_string() => vec![
+            // Single image joke w/ title
+            Selector::parse("article > section > div > figure").unwrap(),
+            // Single cartoon
+            Selector::parse("article > div > div > figure").unwrap(),
+            // Image at top of article
+            Selector::parse("article > header > div > div > figure").unwrap(),
+            // Article body
+            Selector::parse("article .entry-content > *").unwrap(),
+        ],
+        "trofi.github.io".to_string() => vec![
+            Selector::parse("#content").unwrap(),
+        ],
+        "www.redox-os.org".to_string() => vec![
+            Selector::parse("div.content").unwrap(),
+        ],
+        "www.smbc-comics.com".to_string() => vec![
+            Selector::parse("img#cc-comic").unwrap(),
+            Selector::parse("div#aftercomic img").unwrap(),
+        ],
+    ]
+}

server/src/nm.rs

@@ -1,13 +1,929 @@
-use shared::Message;
+use std::{collections::HashMap, fs::File};
 
-use crate::error;
+use letterbox_notmuch::Notmuch;
+use letterbox_shared::compute_color;
+use log::{error, info, warn};
+use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
+use memmap::MmapOptions;
+use sqlx::PgPool;
+use tracing::instrument;
+
+use crate::{
+    compute_offset_limit,
+    error::ServerError,
+    graphql::{
+        Attachment, Body, Corpus, DispositionType, Email, EmailThread, Header, Html, Message,
+        PlainText, Tag, Thread, ThreadSummary, UnhandledContentType,
+    },
+    linkify_html, InlineStyle, Query, SanitizeHtml, Transformer,
+};
+
+const IMAGE_JPEG: &'static str = "image/jpeg";
+const IMAGE_PJPEG: &'static str = "image/pjpeg";
+const IMAGE_PNG: &'static str = "image/png";
+const MESSAGE_RFC822: &'static str = "message/rfc822";
+const MULTIPART_ALTERNATIVE: &'static str = "multipart/alternative";
+const MULTIPART_MIXED: &'static str = "multipart/mixed";
+const MULTIPART_RELATED: &'static str = "multipart/related";
+const TEXT_HTML: &'static str = "text/html";
+const TEXT_PLAIN: &'static str = "text/plain";
+
+const MAX_RAW_MESSAGE_SIZE: usize = 100_000;
+
+fn is_notmuch_query(query: &Query) -> bool {
+    query.is_notmuch || query.corpus == Some(Corpus::Notmuch)
+}
+
+pub fn is_notmuch_thread_or_id(id: &str) -> bool {
+    id.starts_with("id:") || id.starts_with("thread:")
+}
 
 // TODO(wathiede): decide good error type
 pub fn threadset_to_messages(
-    thread_set: notmuch::ThreadSet,
-) -> Result<Vec<Message>, error::ServerError> {
+    thread_set: letterbox_notmuch::ThreadSet,
+) -> Result<Vec<Message>, ServerError> {
     for t in thread_set.0 {
         for _tn in t.0 {}
     }
     Ok(Vec::new())
 }
+
+#[instrument(name="nm::count", skip_all, fields(query=%query))]
+pub async fn count(nm: &Notmuch, query: &Query) -> Result<usize, ServerError> {
+    if !is_notmuch_query(query) {
+        return Ok(0);
+    }
+    let query = query.to_notmuch();
+    Ok(nm.count(&query)?)
+}
+
+#[instrument(name="nm::search", skip_all, fields(query=%query))]
+pub async fn search(
+    nm: &Notmuch,
+    after: Option<i32>,
+    before: Option<i32>,
+    first: Option<i32>,
+    last: Option<i32>,
+    query: &Query,
+) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
+    if !is_notmuch_query(query) {
+        return Ok(Vec::new());
+    }
+    let query = query.to_notmuch();
+    let (offset, mut limit) = compute_offset_limit(after, before, first, last);
+    if before.is_none() {
+        // When searching forward, the +1 is to see if there are more pages of data available.
+        // Searching backwards implies there's more pages forward, because the value represented by
+        // `before` is on the next page.
+        limit = limit + 1;
+    }
+    Ok(nm
+        .search(&query, offset as usize, limit as usize)?
+        .0
+        .into_iter()
+        .enumerate()
+        .map(|(i, ts)| {
+            (
+                offset + i as i32,
+                ThreadSummary {
+                    thread: format!("thread:{}", ts.thread),
+                    timestamp: ts.timestamp,
+                    date_relative: ts.date_relative,
+                    matched: ts.matched,
+                    total: ts.total,
+                    authors: ts.authors,
+                    subject: ts.subject,
+                    tags: ts.tags,
+                    corpus: Corpus::Notmuch,
+                },
+            )
+        })
+        .collect())
+}
+
+#[instrument(name="nm::tags", skip_all, fields(needs_unread=needs_unread))]
+pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
+    let unread_msg_cnt: HashMap<String, usize> = if needs_unread {
+        // 10000 is an arbitrary number, if there's more than 10k unread messages, we'll
+        //  get an inaccurate count.
+        nm.search("is:unread", 0, 10000)?
+            .0
+            .iter()
+            .fold(HashMap::new(), |mut m, ts| {
+                ts.tags.iter().for_each(|t| {
+                    m.entry(t.clone()).and_modify(|c| *c += 1).or_insert(1);
+                });
+                m
+            })
+    } else {
+        HashMap::new()
+    };
+    let tags: Vec<_> = nm
+        .tags()?
+        .into_iter()
+        .map(|tag| {
+            let hex = compute_color(&tag);
+            let unread = if needs_unread {
+                *unread_msg_cnt.get(&tag).unwrap_or(&0)
+            } else {
+                0
+            };
+            Tag {
+                name: tag,
+                fg_color: "white".to_string(),
+                bg_color: hex,
+                unread,
+            }
+        })
+        .chain(
+            nm.unread_recipients()?
+                .into_iter()
+                .filter_map(|(name, unread)| {
+                    let Some(idx) = name.find('@') else {
+                        return None;
+                    };
+                    let name = format!("{}/{}", &name[idx..], &name[..idx]);
+                    let bg_color = compute_color(&name);
+                    Some(Tag {
+                        name,
+                        fg_color: "white".to_string(),
+                        bg_color,
+                        unread,
+                    })
+                }),
+        )
+        .collect();
+    Ok(tags)
+}
+
+#[instrument(name="nm::thread", skip_all, fields(thread_id=thread_id))]
+pub async fn thread(
+    nm: &Notmuch,
+    pool: &PgPool,
+    thread_id: String,
+    debug_content_tree: bool,
+) -> Result<Thread, ServerError> {
+    // TODO(wathiede): normalize all email addresses through an address book with preferred
+    // display names (that default to the most commonly seen name).
+    let mut messages = Vec::new();
+    for (path, id) in std::iter::zip(nm.files(&thread_id)?, nm.message_ids(&thread_id)?) {
+        let tags = nm.tags_for_query(&format!("id:{id}"))?;
+        let file = File::open(&path)?;
+        let mmap = unsafe { MmapOptions::new().map(&file)? };
+        let m = parse_mail(&mmap)?;
+        let from = email_addresses(&path, &m, "from")?;
+        let mut from = match from.len() {
+            0 => None,
+            1 => from.into_iter().next(),
+            _ => {
+                warn!(
+                    "Got {} from addresses in message, truncating: {:?}",
+                    from.len(),
+                    from
+                );
+                from.into_iter().next()
+            }
+        };
+        match from.as_mut() {
+            Some(from) => {
+                if let Some(addr) = from.addr.as_mut() {
+                    let photo_url = photo_url_for_email_address(&pool, &addr).await?;
+                    from.photo_url = photo_url;
+                }
+            }
+            _ => (),
+        }
+
+        let to = email_addresses(&path, &m, "to")?;
+        let cc = email_addresses(&path, &m, "cc")?;
+        let delivered_to = email_addresses(&path, &m, "delivered-to")?.pop();
+        let x_original_to = email_addresses(&path, &m, "x-original-to")?.pop();
+        let subject = m.headers.get_first_value("subject");
+        let timestamp = m
+            .headers
+            .get_first_value("date")
+            .and_then(|d| mailparse::dateparse(&d).ok());
+        let cid_prefix = letterbox_shared::urls::cid_prefix(None, &id);
+        let base_url = None;
+        let mut part_addr = Vec::new();
+        part_addr.push(id.to_string());
+        let body = match extract_body(&m, &mut part_addr)? {
+            Body::PlainText(PlainText { text, content_tree }) => {
+                let text = if text.len() > MAX_RAW_MESSAGE_SIZE {
+                    format!(
+                        "{}...\n\nMESSAGE WAS TRUNCATED @ {} bytes",
+                        &text[..MAX_RAW_MESSAGE_SIZE],
+                        MAX_RAW_MESSAGE_SIZE
+                    )
+                } else {
+                    text
+                };
+
+                Body::Html(Html {
+                    html: {
+                        let body_tranformers: Vec<Box<dyn Transformer>> = vec![
+                            Box::new(InlineStyle),
+                            Box::new(SanitizeHtml {
+                                cid_prefix: &cid_prefix,
+                                base_url: &base_url,
+                            }),
+                        ];
+                        let mut html = linkify_html(&text.trim_matches('\n'));
+                        for t in body_tranformers.iter() {
+                            if t.should_run(&None, &html) {
+                                html = t.transform(&None, &html).await?;
+                            }
+                        }
+
+                        format!(
+                            r#"<p class="view-part-text-plain font-mono whitespace-pre-line">{}</p>"#,
+                            // Trim newlines to prevent excessive white space at the beginning/end of
+                            // presenation. Leave tabs and spaces incase plain text attempts to center a
+                            // header on the first line.
+                            html
+                        )
+                    },
+                    content_tree: if debug_content_tree {
+                        render_content_type_tree(&m)
+                    } else {
+                        content_tree
+                    },
+                })
+            }
+            Body::Html(Html {
+                mut html,
+                content_tree,
+            }) => Body::Html(Html {
+                html: {
+                    let body_tranformers: Vec<Box<dyn Transformer>> = vec![
+                        // TODO: this breaks things like emails from calendar
+                        //Box::new(InlineStyle),
+                        Box::new(SanitizeHtml {
+                            cid_prefix: &cid_prefix,
+                            base_url: &base_url,
+                        }),
+                    ];
+                    for t in body_tranformers.iter() {
+                        if t.should_run(&None, &html) {
+                            html = t.transform(&None, &html).await?;
+                        }
+                    }
+                    html
+                },
+
+                content_tree: if debug_content_tree {
+                    render_content_type_tree(&m)
+                } else {
+                    content_tree
+                },
+            }),
+
+            Body::UnhandledContentType(UnhandledContentType { content_tree, .. }) => {
+                let body_start = mmap
+                    .windows(2)
+                    .take(20_000)
+                    .position(|w| w == b"\n\n")
+                    .unwrap_or(0);
+                let body = mmap[body_start + 2..].to_vec();
+                Body::UnhandledContentType(UnhandledContentType {
+                    text: String::from_utf8(body)?,
+                    content_tree: if debug_content_tree {
+                        render_content_type_tree(&m)
+                    } else {
+                        content_tree
+                    },
+                })
+            }
+        };
+        let headers = m
+            .headers
+            .iter()
+            .map(|h| Header {
+                key: h.get_key(),
+                value: h.get_value(),
+            })
+            .collect();
+        // TODO(wathiede): parse message and fill out attachments
+        let attachments = extract_attachments(&m, &id)?;
+        messages.push(Message {
+            id: format!("id:{id}"),
+            from,
+            to,
+            cc,
+            subject,
+            tags,
+            timestamp,
+            headers,
+            body,
+            path,
+            attachments,
+            delivered_to,
+            x_original_to,
+        });
+    }
+    messages.reverse();
+    // Find the first subject that's set. After reversing the vec, this should be the oldest
+    // message.
+    let subject: String = messages
+        .iter()
+        .skip_while(|m| m.subject.is_none())
+        .next()
+        .and_then(|m| m.subject.clone())
+        .unwrap_or("(NO SUBJECT)".to_string());
+    Ok(Thread::Email(EmailThread {
+        thread_id,
+        subject,
+        messages,
+    }))
+}
+
+fn email_addresses(
+    path: &str,
+    m: &ParsedMail,
+    header_name: &str,
+) -> Result<Vec<Email>, ServerError> {
+    let mut addrs = Vec::new();
+    for header_value in m.headers.get_all_values(header_name) {
+        match mailparse::addrparse(&header_value) {
+            Ok(mal) => {
+                for ma in mal.into_inner() {
+                    match ma {
+                        mailparse::MailAddr::Group(gi) => {
+                            if !gi.group_name.contains("ndisclosed") {
+                                println!("[{path}][{header_name}] Group: {gi}");
+                            }
+                        }
+                        mailparse::MailAddr::Single(s) => addrs.push(Email {
+                            name: s.display_name,
+                            addr: Some(s.addr),
+                            photo_url: None,
+                        }), //println!("Single: {s}"),
+                    }
+                }
+            }
+            Err(_) => {
+                let v = header_value;
+                if v.matches('@').count() == 1 {
+                    if v.matches('<').count() == 1 && v.ends_with('>') {
+                        let idx = v.find('<').unwrap();
+                        let addr = &v[idx + 1..v.len() - 1].trim();
+                        let name = &v[..idx].trim();
+                        addrs.push(Email {
+                            name: Some(name.to_string()),
+                            addr: Some(addr.to_string()),
+                            photo_url: None,
+                        });
+                    }
+                } else {
+                    addrs.push(Email {
+                        name: Some(v),
+                        addr: None,
+                        photo_url: None,
+                    });
+                }
+            }
+        }
+    }
+    Ok(addrs)
+}
+
+pub fn cid_attachment_bytes(nm: &Notmuch, id: &str, cid: &str) -> Result<Attachment, ServerError> {
+    let files = nm.files(id)?;
+    let Some(path) = files.first() else {
+        warn!("failed to find files for message {id}");
+        return Err(ServerError::PartNotFound);
+    };
+    let file = File::open(&path)?;
+    let mmap = unsafe { MmapOptions::new().map(&file)? };
+    let m = parse_mail(&mmap)?;
+    if let Some(attachment) = walk_attachments(&m, |sp, _cur_idx| {
+        info!("{cid} {:?}", get_content_id(&sp.headers));
+        if let Some(h_cid) = get_content_id(&sp.headers) {
+            let h_cid = &h_cid[1..h_cid.len() - 1];
+            if h_cid == cid {
+                let attachment = extract_attachment(&sp, id, &[]).unwrap_or(Attachment {
+                    ..Attachment::default()
+                });
+                return Some(attachment);
+            }
+        }
+        None
+    }) {
+        return Ok(attachment);
+    }
+
+    Err(ServerError::PartNotFound)
+}
+
+pub fn attachment_bytes(nm: &Notmuch, id: &str, idx: &[usize]) -> Result<Attachment, ServerError> {
+    let files = nm.files(id)?;
+    let Some(path) = files.first() else {
+        warn!("failed to find files for message {id}");
+        return Err(ServerError::PartNotFound);
+    };
+    let file = File::open(&path)?;
+    let mmap = unsafe { MmapOptions::new().map(&file)? };
+    let m = parse_mail(&mmap)?;
+    if let Some(attachment) = walk_attachments(&m, |sp, cur_idx| {
+        if cur_idx == idx {
+            let attachment = extract_attachment(&sp, id, idx).unwrap_or(Attachment {
+                ..Attachment::default()
+            });
+            return Some(attachment);
+        }
+        None
+    }) {
+        return Ok(attachment);
+    }
+
+    Err(ServerError::PartNotFound)
+}
+
+fn extract_body(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+    let body = m.get_body()?;
+    let ret = match m.ctype.mimetype.as_str() {
+        TEXT_PLAIN => return Ok(Body::text(body)),
+        TEXT_HTML => return Ok(Body::html(body)),
+        MULTIPART_MIXED => extract_mixed(m, part_addr),
+        MULTIPART_ALTERNATIVE => extract_alternative(m, part_addr),
+        MULTIPART_RELATED => extract_related(m, part_addr),
+        _ => extract_unhandled(m),
+    };
+    if let Err(err) = ret {
+        error!("Failed to extract body: {err:?}");
+        return Ok(extract_unhandled(m)?);
+    }
+    ret
+}
+
+fn extract_unhandled(m: &ParsedMail) -> Result<Body, ServerError> {
+    let msg = format!(
+        "Unhandled body content type:\n{}\n{}",
+        render_content_type_tree(m),
+        m.get_body()?,
+    );
+    Ok(Body::UnhandledContentType(UnhandledContentType {
+        text: msg,
+        content_tree: render_content_type_tree(m),
+    }))
+}
+
+// multipart/alternative defines multiple representations of the same message, and clients should
+// show the fanciest they can display. For this program, the priority is text/html, text/plain,
+// then give up.
+fn extract_alternative(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+    let handled_types = vec![
+        MULTIPART_ALTERNATIVE,
+        MULTIPART_MIXED,
+        MULTIPART_RELATED,
+        TEXT_HTML,
+        TEXT_PLAIN,
+    ];
+    for sp in &m.subparts {
+        if sp.ctype.mimetype.as_str() == MULTIPART_ALTERNATIVE {
+            return extract_alternative(sp, part_addr);
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype.as_str() == MULTIPART_MIXED {
+            return extract_mixed(sp, part_addr);
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype.as_str() == MULTIPART_RELATED {
+            return extract_related(sp, part_addr);
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype.as_str() == TEXT_HTML {
+            let body = sp.get_body()?;
+            return Ok(Body::html(body));
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype.as_str() == TEXT_PLAIN {
+            let body = sp.get_body()?;
+            return Ok(Body::text(body));
+        }
+    }
+    Err(ServerError::StringError(format!(
+        "extract_alternative failed to find suitable subpart, searched: {:?}",
+        handled_types
+    )))
+}
+
+// multipart/mixed defines multiple types of context all of which should be presented to the user
+// 'serially'.
+fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+    //todo!("add some sort of visual indicator there are unhandled types, i.e. .ics files");
+    let handled_types = vec![
+        IMAGE_JPEG,
+        IMAGE_PJPEG,
+        IMAGE_PNG,
+        MESSAGE_RFC822,
+        MULTIPART_ALTERNATIVE,
+        MULTIPART_RELATED,
+        TEXT_HTML,
+        TEXT_PLAIN,
+    ];
+    let mut unhandled_types: Vec<_> = m
+        .subparts
+        .iter()
+        .map(|sp| sp.ctype.mimetype.as_str())
+        .filter(|mt| !handled_types.contains(&mt))
+        .collect();
+    unhandled_types.sort();
+    if !unhandled_types.is_empty() {
+        warn!("{MULTIPART_MIXED} contains the following unhandled mimetypes {unhandled_types:?}");
+    }
+    let mut parts = Vec::new();
+    for (idx, sp) in m.subparts.iter().enumerate() {
+        part_addr.push(idx.to_string());
+        match sp.ctype.mimetype.as_str() {
+            MESSAGE_RFC822 => parts.push(extract_rfc822(&sp, part_addr)?),
+            MULTIPART_RELATED => parts.push(extract_related(sp, part_addr)?),
+            MULTIPART_ALTERNATIVE => parts.push(extract_alternative(sp, part_addr)?),
+            TEXT_PLAIN => parts.push(Body::text(sp.get_body()?)),
+            TEXT_HTML => parts.push(Body::html(sp.get_body()?)),
+            IMAGE_PJPEG | IMAGE_JPEG | IMAGE_PNG => {
+                let pcd = sp.get_content_disposition();
+                let filename = pcd
+                    .params
+                    .get("filename")
+                    .map(|s| s.clone())
+                    .unwrap_or("".to_string());
+                // Only add inline images, attachments are handled as an attribute of the top level Message and rendered separate client-side.
+                if pcd.disposition == mailparse::DispositionType::Inline {
+                    // TODO: make URL generation more programatic based on what the frontend has
+                    // mapped
+                    parts.push(Body::html(format!(
+                        r#"<img src="/api/view/attachment/{}/{}/{filename}">"#,
+                        part_addr[0],
+                        part_addr
+                            .iter()
+                            .skip(1)
+                            .map(|i| i.to_string())
+                            .collect::<Vec<_>>()
+                            .join(".")
+                    )));
+                }
+            }
+            mt => parts.push(unhandled_html(MULTIPART_MIXED, mt)),
+        }
+        part_addr.pop();
+    }
+    Ok(flatten_body_parts(&parts))
+}
+
+fn unhandled_html(parent_type: &str, child_type: &str) -> Body {
+    Body::Html(Html {
+        html: format!(
+            r#"
+<div class="p-4 error">
+Unhandled mimetype {child_type} in a {parent_type} message
+</div>
+    "#
+        ),
+        content_tree: String::new(),
+    })
+}
+fn flatten_body_parts(parts: &[Body]) -> Body {
+    let html = parts
+        .iter()
+        .map(|p| match p {
+            Body::PlainText(PlainText { text, .. }) => {
+                format!(
+                    r#"<p class="view-part-text-plain font-mono whitespace-pre-line">{}</p>"#,
+                    // Trim newlines to prevent excessive white space at the beginning/end of
+                    // presenation. Leave tabs and spaces incase plain text attempts to center a
+                    // header on the first line.
+                    linkify_html(&html_escape::encode_text(text).trim_matches('\n'))
+                )
+            }
+            Body::Html(Html { html, .. }) => html.clone(),
+            Body::UnhandledContentType(UnhandledContentType { text, .. }) => {
+                error!("text len {}", text.len());
+                format!(
+                    r#"<p class="view-part-unhandled">{}</p>"#,
+                    // Trim newlines to prevent excessive white space at the beginning/end of
+                    // presenation. Leave tabs and spaces incase plain text attempts to center a
+                    // header on the first line.
+                    linkify_html(&html_escape::encode_text(text).trim_matches('\n'))
+                )
+            }
+        })
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    info!("flatten_body_parts {}", parts.len());
+    Body::html(html)
+}
+
+fn extract_related(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+    // TODO(wathiede): collect related things and change return type to new Body arm.
+    let handled_types = vec![
+        MULTIPART_ALTERNATIVE,
+        TEXT_HTML,
+        TEXT_PLAIN,
+        IMAGE_JPEG,
+        IMAGE_PJPEG,
+        IMAGE_PNG,
+    ];
+    let mut unhandled_types: Vec<_> = m
+        .subparts
+        .iter()
+        .map(|sp| sp.ctype.mimetype.as_str())
+        .filter(|mt| !handled_types.contains(&mt))
+        .collect();
+    unhandled_types.sort();
+    if !unhandled_types.is_empty() {
+        warn!("{MULTIPART_RELATED} contains the following unhandled mimetypes {unhandled_types:?}");
+    }
+
+    for (i, sp) in m.subparts.iter().enumerate() {
+        if sp.ctype.mimetype == IMAGE_PNG
+            || sp.ctype.mimetype == IMAGE_JPEG
+            || sp.ctype.mimetype == IMAGE_PJPEG
+        {
+            info!("sp.ctype {:#?}", sp.ctype);
+            //info!("sp.headers {:#?}", sp.headers);
+            if let Some(cid) = sp.headers.get_first_value("Content-Id") {
+                let mut part_id = part_addr.clone();
+                part_id.push(i.to_string());
+                info!("cid: {cid} part_id {part_id:?}");
+            }
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype == MULTIPART_ALTERNATIVE {
+            return extract_alternative(m, part_addr);
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype == TEXT_HTML {
+            let body = sp.get_body()?;
+            return Ok(Body::html(body));
+        }
+    }
+    for sp in &m.subparts {
+        if sp.ctype.mimetype == TEXT_PLAIN {
+            let body = sp.get_body()?;
+            return Ok(Body::text(body));
+        }
+    }
+    Err(ServerError::StringError(format!(
+        "extract_related failed to find suitable subpart, searched: {:?}",
+        handled_types
+    )))
+}
+
+fn walk_attachments<T, F: Fn(&ParsedMail, &[usize]) -> Option<T> + Copy>(
+    m: &ParsedMail,
+    visitor: F,
+) -> Option<T> {
+    let mut cur_addr = Vec::new();
+    walk_attachments_inner(m, visitor, &mut cur_addr)
+}
+
+fn walk_attachments_inner<T, F: Fn(&ParsedMail, &[usize]) -> Option<T> + Copy>(
+    m: &ParsedMail,
+    visitor: F,
+    cur_addr: &mut Vec<usize>,
+) -> Option<T> {
+    for (idx, sp) in m.subparts.iter().enumerate() {
+        cur_addr.push(idx);
+        let val = visitor(sp, &cur_addr);
+        if val.is_some() {
+            return val;
+        }
+        let val = walk_attachments_inner(sp, visitor, cur_addr);
+        if val.is_some() {
+            return val;
+        }
+        cur_addr.pop();
+    }
+    None
+}
+
+// TODO(wathiede): make this walk_attachments that takes a closure.
+// Then implement one closure for building `Attachment` and imlement another that can be used to
+// get the bytes for serving attachments of HTTP
+fn extract_attachments(m: &ParsedMail, id: &str) -> Result<Vec<Attachment>, ServerError> {
+    let mut attachments = Vec::new();
+    for (idx, sp) in m.subparts.iter().enumerate() {
+        if let Some(attachment) = extract_attachment(sp, id, &[idx]) {
+            // Filter out inline attachements, they're flattened into the body of the message.
+            if attachment.disposition == DispositionType::Attachment {
+                attachments.push(attachment);
+            }
+        }
+    }
+    Ok(attachments)
+}
+
+fn extract_attachment(m: &ParsedMail, id: &str, idx: &[usize]) -> Option<Attachment> {
+    let pcd = m.get_content_disposition();
+    let pct = m
+        .get_headers()
+        .get_first_value("Content-Type")
+        .map(|s| parse_content_type(&s));
+    let filename = match (
+        pcd.params.get("filename").map(|f| f.clone()),
+        pct.map(|pct| pct.params.get("name").map(|f| f.clone())),
+    ) {
+        // Use filename from Content-Disposition
+        (Some(filename), _) => filename,
+        // Use filename from Content-Type
+        (_, Some(Some(name))) => name,
+        // No known filename, assume it's not an attachment
+        _ => return None,
+    };
+    info!("filename {filename}");
+
+    // TODO: grab this from somewhere
+    let content_id = None;
+    let bytes = match m.get_body_raw() {
+        Ok(bytes) => bytes,
+        Err(err) => {
+            error!("failed to get body for attachment: {err}");
+            return None;
+        }
+    };
+    return Some(Attachment {
+        id: id.to_string(),
+        idx: idx
+            .iter()
+            .map(|i| i.to_string())
+            .collect::<Vec<_>>()
+            .join("."),
+        disposition: pcd.disposition.into(),
+        filename: Some(filename),
+        size: bytes.len(),
+        // TODO: what is the default for ctype?
+        // TODO: do we want to use m.ctype.params for anything?
+        content_type: Some(m.ctype.mimetype.clone()),
+        content_id,
+        bytes,
+    });
+}
+fn email_address_strings(emails: &[Email]) -> Vec<String> {
+    emails
+        .iter()
+        .map(|e| e.to_string())
+        .inspect(|e| info!("e {e}"))
+        .collect()
+}
+
+fn extract_rfc822(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, ServerError> {
+    fn extract_headers(m: &ParsedMail) -> Result<Body, ServerError> {
+        let path = "<in-memory>";
+        let from = email_address_strings(&email_addresses(path, &m, "from")?).join(", ");
+        let to = email_address_strings(&email_addresses(path, &m, "to")?).join(", ");
+        let cc = email_address_strings(&email_addresses(path, &m, "cc")?).join(", ");
+        let date = m.headers.get_first_value("date").unwrap_or(String::new());
+        let subject = m
+            .headers
+            .get_first_value("subject")
+            .unwrap_or(String::new());
+        let text = format!(
+            r#"
+---------- Forwarded message ----------
+From: {from}
+To: {to}
+CC: {cc}
+Date: {date}
+Subject: {subject}
+"#
+        );
+        Ok(Body::text(text))
+    }
+    let inner_body = m.get_body()?;
+    let inner_m = parse_mail(inner_body.as_bytes())?;
+    let headers = extract_headers(&inner_m)?;
+    let body = extract_body(&inner_m, part_addr)?;
+
+    Ok(flatten_body_parts(&[headers, body]))
+}
+
+pub fn get_attachment_filename(header_value: &str) -> &str {
+    info!("get_attachment_filename {header_value}");
+    // Strip last "
+    let v = &header_value[..header_value.len() - 1];
+    if let Some(idx) = v.rfind('"') {
+        &v[idx + 1..]
+    } else {
+        ""
+    }
+}
+
+pub fn get_content_type<'a>(headers: &[MailHeader<'a>]) -> Option<String> {
+    if let Some(v) = headers.get_first_value("Content-Type") {
+        if let Some(idx) = v.find(';') {
+            return Some(v[..idx].to_string());
+        } else {
+            return Some(v);
+        }
+    }
+    None
+}
+
+fn get_content_id<'a>(headers: &[MailHeader<'a>]) -> Option<String> {
+    headers.get_first_value("Content-Id")
+}
+
+fn render_content_type_tree(m: &ParsedMail) -> String {
+    const WIDTH: usize = 4;
+    const SKIP_HEADERS: [&str; 4] = [
+        "Authentication-Results",
+        "DKIM-Signature",
+        "Received",
+        "Received-SPF",
+    ];
+    fn render_ct_rec(m: &ParsedMail, depth: usize) -> String {
+        let mut parts = Vec::new();
+        let msg = format!("{} {}", "-".repeat(depth * WIDTH), m.ctype.mimetype);
+        parts.push(msg);
+        for sp in &m.subparts {
+            parts.push(render_ct_rec(sp, depth + 1))
+        }
+        parts.join("\n")
+    }
+    fn render_rec(m: &ParsedMail, depth: usize) -> String {
+        let mut parts = Vec::new();
+        let msg = format!("{} {}", "-".repeat(depth * WIDTH), m.ctype.mimetype);
+        parts.push(msg);
+        let indent = " ".repeat(depth * WIDTH);
+        if !m.ctype.charset.is_empty() {
+            parts.push(format!("{indent} Character Set: {}", m.ctype.charset));
+        }
+        for (k, v) in m.ctype.params.iter() {
+            parts.push(format!("{indent} {k}: {v}"));
+        }
+        if !m.headers.is_empty() {
+            parts.push(format!("{indent} == headers =="));
+            for h in &m.headers {
+                if h.get_key().starts_with('X') {
+                    continue;
+                }
+                if SKIP_HEADERS.contains(&h.get_key().as_str()) {
+                    continue;
+                }
+
+                parts.push(format!("{indent} {}: {}", h.get_key_ref(), h.get_value()));
+            }
+        }
+        for sp in &m.subparts {
+            parts.push(render_rec(sp, depth + 1))
+        }
+        parts.join("\n")
+    }
+    format!(
+        "Outline:\n{}\n\nDetailed:\n{}\n\nNot showing headers:\n  {}\n  X.*",
+        render_ct_rec(m, 1),
+        render_rec(m, 1),
+        SKIP_HEADERS.join("\n  ")
+    )
+}
+
+#[instrument(name="nm::set_read_status", skip_all, fields(query=%query, unread=unread))]
+pub async fn set_read_status<'ctx>(
+    nm: &Notmuch,
+    query: &Query,
+    unread: bool,
+) -> Result<bool, ServerError> {
+    let uids: Vec<_> = query
+        .uids
+        .iter()
+        .filter(|uid| is_notmuch_thread_or_id(uid))
+        .collect();
+    info!("set_read_status({unread} {uids:?})");
+    for uid in uids {
+        if unread {
+            nm.tag_add("unread", uid)?;
+        } else {
+            nm.tag_remove("unread", uid)?;
+        }
+    }
+    Ok(true)
+}
+
+async fn photo_url_for_email_address(
+    pool: &PgPool,
+    addr: &str,
+) -> Result<Option<String>, ServerError> {
+    let row = sqlx::query!(
+        r#"
+SELECT
+  url
+FROM email_photo ep
+JOIN email_address ea
+ON ep.id = ea.email_photo_id
+WHERE
+  address = $1
+    "#,
+        addr
+    )
+    .fetch_optional(pool)
+    .await?;
+    Ok(row.map(|r| r.url))
+}

server/src/tantivy.rs

@@ -0,0 +1,353 @@
+use std::collections::HashSet;
+
+use log::{debug, error, info, warn};
+use sqlx::{postgres::PgPool, types::time::PrimitiveDateTime};
+use tantivy::{
+    collector::{DocSetCollector, TopDocs},
+    doc, query,
+    query::{AllQuery, BooleanQuery, Occur, QueryParser, TermQuery},
+    schema::{Facet, IndexRecordOption, Value},
+    DocAddress, Index, IndexReader, Searcher, TantivyDocument, TantivyError, Term,
+};
+use tracing::{info_span, instrument, Instrument};
+
+use crate::{
+    compute_offset_limit,
+    error::ServerError,
+    graphql::{Corpus, ThreadSummary},
+    newsreader::{extract_thread_id, is_newsreader_thread},
+    thread_summary_from_row, Query, ThreadSummaryRecord,
+};
+
+pub fn is_tantivy_query(query: &Query) -> bool {
+    query.is_tantivy || query.corpus == Some(Corpus::Tantivy)
+}
+pub struct TantivyConnection {
+    db_path: String,
+    index: Index,
+    reader: IndexReader,
+}
+
+fn get_index(db_path: &str) -> Result<Index, TantivyError> {
+    Ok(match Index::open_in_dir(db_path) {
+        Ok(idx) => idx,
+        Err(err) => {
+            warn!("Failed to open {db_path}: {err}");
+            create_news_db(db_path)?;
+            Index::open_in_dir(db_path)?
+        }
+    })
+}
+
+impl TantivyConnection {
+    pub fn new(tantivy_db_path: &str) -> Result<TantivyConnection, TantivyError> {
+        let index = get_index(tantivy_db_path)?;
+        let reader = index.reader()?;
+
+        Ok(TantivyConnection {
+            db_path: tantivy_db_path.to_string(),
+            index,
+            reader,
+        })
+    }
+    #[instrument(name = "tantivy::refresh", skip_all)]
+    pub async fn refresh(&self, pool: &PgPool) -> Result<(), ServerError> {
+        let start_time = std::time::Instant::now();
+        let p_uids: Vec<_> = sqlx::query_file!("sql/all-uids.sql")
+            .fetch_all(pool)
+            .instrument(info_span!("postgres query"))
+            .await?
+            .into_iter()
+            .map(|r| r.uid)
+            .collect();
+        info!(
+            "refresh from postgres got {} uids in {}",
+            p_uids.len(),
+            start_time.elapsed().as_secs_f32()
+        );
+
+        let t_span = info_span!("tantivy query");
+        let _enter = t_span.enter();
+        let start_time = std::time::Instant::now();
+        let (searcher, _query) = self.searcher_and_query(&Query::default())?;
+        let docs = searcher.search(&AllQuery, &DocSetCollector)?;
+        let uid = self.index.schema().get_field("uid")?;
+        let t_uids: Vec<_> = docs
+            .into_iter()
+            .map(|doc_address| {
+                searcher
+                    .doc(doc_address)
+                    .map(|doc: TantivyDocument| {
+                        debug!("doc: {doc:#?}");
+                        doc.get_first(uid)
+                            .expect("uid")
+                            .as_str()
+                            .expect("as_str")
+                            .to_string()
+                    })
+                    .expect("searcher.doc")
+            })
+            .collect();
+        drop(_enter);
+
+        info!(
+            "refresh tantivy got {} uids in {}",
+            t_uids.len(),
+            start_time.elapsed().as_secs_f32()
+        );
+        let t_set: HashSet<_> = t_uids.into_iter().collect();
+        let need: Vec<_> = p_uids
+            .into_iter()
+            .filter(|uid| !t_set.contains(uid.as_str()))
+            .collect();
+        if !need.is_empty() {
+            info!(
+                "need to reindex {} uids: {:?}...",
+                need.len(),
+                &need[..need.len().min(10)]
+            );
+        }
+        let batch_size = 1000;
+        let uids: Vec<_> = need[..need.len().min(batch_size)]
+            .into_iter()
+            .cloned()
+            .collect();
+        self.reindex_uids(pool, &uids).await
+    }
+    #[instrument(skip(self, pool))]
+    async fn reindex_uids(&self, pool: &PgPool, uids: &[String]) -> Result<(), ServerError> {
+        if uids.is_empty() {
+            return Ok(());
+        }
+        // TODO: add SlurpContents and convert HTML to text
+
+        let pool: &PgPool = pool;
+
+        let mut index_writer = self.index.writer(50_000_000)?;
+        let schema = self.index.schema();
+        let site = schema.get_field("site")?;
+        let title = schema.get_field("title")?;
+        let summary = schema.get_field("summary")?;
+        let link = schema.get_field("link")?;
+        let date = schema.get_field("date")?;
+        let is_read = schema.get_field("is_read")?;
+        let uid = schema.get_field("uid")?;
+        let id = schema.get_field("id")?;
+        let tag = schema.get_field("tag")?;
+
+        info!("reindexing {} posts", uids.len());
+        let rows = sqlx::query_file_as!(PostgresDoc, "sql/posts-from-uids.sql", uids)
+            .fetch_all(pool)
+            .await?;
+
+        if uids.len() != rows.len() {
+            error!(
+                "Had {} uids and only got {} rows: uids {uids:?}",
+                uids.len(),
+                rows.len()
+            );
+        }
+        for r in rows {
+            let id_term = Term::from_field_text(uid, &r.uid);
+            index_writer.delete_term(id_term);
+            let slug = r.site;
+            let tag_facet = Facet::from(&format!("/News/{slug}"));
+            index_writer.add_document(doc!(
+                site => slug.clone(),
+                title => r.title,
+                // TODO: clean and extract text from HTML
+                summary => r.summary,
+                link => r.link,
+                date => tantivy::DateTime::from_primitive(r.date),
+                is_read => r.is_read,
+                uid => r.uid,
+                id => r.id as u64,
+                tag => tag_facet,
+            ))?;
+        }
+
+        info_span!("IndexWriter.commit").in_scope(|| index_writer.commit())?;
+        info_span!("IndexReader.reload").in_scope(|| self.reader.reload())?;
+        Ok(())
+    }
+    #[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=%query))]
+    pub async fn reindex_thread(&self, pool: &PgPool, query: &Query) -> Result<(), ServerError> {
+        let uids: Vec<_> = query
+            .uids
+            .iter()
+            .filter(|uid| is_newsreader_thread(uid))
+            .map(|uid| extract_thread_id(uid).to_string())
+            .collect();
+        Ok(self.reindex_uids(pool, &uids).await?)
+    }
+    #[instrument(name = "tantivy::reindex_all", skip_all)]
+    pub async fn reindex_all(&self, pool: &PgPool) -> Result<(), ServerError> {
+        let rows = sqlx::query_file!("sql/all-posts.sql")
+            .fetch_all(pool)
+            .await?;
+
+        let uids: Vec<String> = rows.into_iter().map(|r| r.uid).collect();
+        self.reindex_uids(pool, &uids).await?;
+        Ok(())
+    }
+    fn searcher_and_query(
+        &self,
+        query: &Query,
+    ) -> Result<(Searcher, Box<dyn query::Query>), ServerError> {
+        // TODO: only create one reader
+        // From https://tantivy-search.github.io/examples/basic_search.html
+        // "For a search server you will typically create one reader for the entire lifetime of
+        // your program, and acquire a new searcher for every single request."
+        //
+        // I think there's some challenge in making the reader work if we reindex, so reader my
+        // need to be stored indirectly, and be recreated on reindex
+        // I think creating a reader takes 200-300 ms.
+        let schema = self.index.schema();
+        let searcher = self.reader.searcher();
+        let title = schema.get_field("title")?;
+        let summary = schema.get_field("summary")?;
+        let query_parser = QueryParser::for_index(&self.index, vec![title, summary]);
+        // Tantivy uses '*' to match all docs, not empty string
+        let term = &query.remainder.join(" ");
+        let term = if term.is_empty() { "*" } else { term };
+        info!("query_parser('{term}')");
+
+        let tantivy_query = query_parser.parse_query(&term)?;
+
+        let tag = schema.get_field("tag")?;
+        let is_read = schema.get_field("is_read")?;
+        let mut terms = vec![(Occur::Must, tantivy_query)];
+        for t in &query.tags {
+            let facet = Facet::from(&format!("/{t}"));
+            let facet_term = Term::from_facet(tag, &facet);
+            let facet_term_query = Box::new(TermQuery::new(facet_term, IndexRecordOption::Basic));
+            terms.push((Occur::Must, facet_term_query));
+        }
+        if query.unread_only {
+            info!("searching for unread only");
+            let term = Term::from_field_bool(is_read, false);
+            terms.push((
+                Occur::Must,
+                Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+            ));
+        }
+        let search_query = BooleanQuery::new(terms);
+        Ok((searcher, Box::new(search_query)))
+    }
+
+    #[instrument(name="tantivy::count", skip_all, fields(query=%query))]
+    pub async fn count(&self, query: &Query) -> Result<usize, ServerError> {
+        if !is_tantivy_query(query) {
+            return Ok(0);
+        }
+        info!("tantivy::count {query:?}");
+        use tantivy::collector::Count;
+        let (searcher, query) = self.searcher_and_query(&query)?;
+        Ok(searcher.search(&query, &Count)?)
+    }
+    #[instrument(name="tantivy::search", skip_all, fields(query=%query))]
+    pub async fn search(
+        &self,
+        pool: &PgPool,
+        after: Option<i32>,
+        before: Option<i32>,
+        first: Option<i32>,
+        last: Option<i32>,
+        query: &Query,
+    ) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
+        if !is_tantivy_query(query) {
+            return Ok(Vec::new());
+        }
+        let (offset, mut limit) = compute_offset_limit(after, before, first, last);
+        if before.is_none() {
+            // When searching forward, the +1 is to see if there are more pages of data available.
+            // Searching backwards implies there's more pages forward, because the value represented by
+            // `before` is on the next page.
+            limit = limit + 1;
+        }
+
+        let (searcher, search_query) = self.searcher_and_query(&query)?;
+        info!("Tantivy::search(query '{query:?}', off {offset}, lim {limit}, search_query {search_query:?})");
+        let top_docs = searcher.search(
+            &search_query,
+            &TopDocs::with_limit(limit as usize)
+                .and_offset(offset as usize)
+                .order_by_u64_field("date", tantivy::index::Order::Desc),
+        )?;
+        info!("search found {} docs", top_docs.len());
+        let uid = self.index.schema().get_field("uid")?;
+        let uids = top_docs
+            .into_iter()
+            .map(|(_, doc_address): (u64, DocAddress)| {
+                searcher.doc(doc_address).map(|doc: TantivyDocument| {
+                    debug!("doc: {doc:#?}");
+                    doc.get_first(uid)
+                        .expect("doc missing uid")
+                        .as_str()
+                        .expect("doc str missing")
+                        .to_string()
+                })
+            })
+            .collect::<Result<Vec<String>, TantivyError>>()?;
+
+        //let uids = format!("'{}'", uids.join("','"));
+        info!("uids {uids:?}");
+        let rows = sqlx::query_file!("sql/threads-from-uid.sql", &uids as &[String])
+            .fetch_all(pool)
+            .await?;
+        let mut res = Vec::new();
+        info!("found {} hits joining w/ tantivy", rows.len());
+        for (i, r) in rows.into_iter().enumerate() {
+            res.push((
+                i as i32 + offset,
+                thread_summary_from_row(ThreadSummaryRecord {
+                    site: r.site,
+                    date: r.date,
+                    is_read: r.is_read,
+                    title: r.title,
+                    uid: r.uid,
+                    name: r.name,
+                    corpus: Corpus::Tantivy,
+                })
+                .await,
+            ));
+        }
+        Ok(res)
+    }
+    pub fn drop_and_load_index(&self) -> Result<(), TantivyError> {
+        create_news_db(&self.db_path)
+    }
+}
+
+fn create_news_db(tantivy_db_path: &str) -> Result<(), TantivyError> {
+    info!("create_news_db");
+    // Don't care if directory didn't exist
+    let _ = std::fs::remove_dir_all(tantivy_db_path);
+    std::fs::create_dir_all(tantivy_db_path)?;
+    use tantivy::schema::*;
+    let mut schema_builder = Schema::builder();
+    schema_builder.add_text_field("site", STRING | STORED);
+    schema_builder.add_text_field("title", TEXT | STORED);
+    schema_builder.add_text_field("summary", TEXT);
+    schema_builder.add_text_field("link", STRING | STORED);
+    schema_builder.add_date_field("date", FAST | INDEXED | STORED);
+    schema_builder.add_bool_field("is_read", FAST | INDEXED | STORED);
+    schema_builder.add_text_field("uid", STRING | STORED);
+    schema_builder.add_u64_field("id", FAST);
+    schema_builder.add_facet_field("tag", FacetOptions::default());
+
+    let schema = schema_builder.build();
+    Index::create_in_dir(tantivy_db_path, schema)?;
+    Ok(())
+}
+
+struct PostgresDoc {
+    site: String,
+    title: String,
+    summary: String,
+    link: String,
+    date: PrimitiveDateTime,
+    is_read: bool,
+    uid: String,
+    id: i32,
+}

server/static/graphql-playground.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset=utf-8 />
+  <meta name="viewport" content="user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, minimal-ui">
+  <title>GraphQL Playground</title>
+  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/css/index.css" />
+  <link rel="shortcut icon" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/favicon.png" />
+  <script src="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/js/middleware.js"></script>
+</head>
+
+<body>
+  <div id="root">
+    <style>
+      body {
+        background-color: rgb(23, 42, 58);
+        font-family: Open Sans, sans-serif;
+        height: 90vh;
+      }
+
+      #root {
+        height: 100%;
+        width: 100%;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+      }
+
+      .loading {
+        font-size: 32px;
+        font-weight: 200;
+        color: rgba(255, 255, 255, .6);
+        margin-left: 20px;
+      }
+
+      img {
+        width: 78px;
+        height: 78px;
+      }
+
+      .title {
+        font-weight: 400;
+      }
+    </style>
+    <img src='//cdn.jsdelivr.net/npm/graphql-playground-react/build/logo.png' alt=''>
+    <div class="loading"> Loading
+      <span class="title">GraphQL Playground</span>
+    </div>
+  </div>
+  <script>window.addEventListener('load', function (event) {
+      GraphQLPlayground.init(document.getElementById('root'), {
+        // options as 'endpoint' belong here
+        endpoint: "/api/graphql",
+      })
+    })</script>
+</body>
+
+</html>

server/static/vars.css

@@ -0,0 +1,42 @@
+:root {
+    --active-brightness: 0.85;
+    --border-radius: 5px;
+    --box-shadow: 2px 2px 10px;
+    --color-accent: #118bee15;
+    --color-bg: #fff;
+    --color-bg-secondary: #e9e9e9;
+    --color-link: #118bee;
+    --color-secondary: #920de9;
+    --color-secondary-accent: #920de90b;
+    --color-shadow: #f4f4f4;
+    --color-table: #118bee;
+    --color-text: #000;
+    --color-text-secondary: #999;
+    --color-scrollbar: #cacae8;
+    --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+    --hover-brightness: 1.2;
+    --justify-important: center;
+    --justify-normal: left;
+    --line-height: 1.5;
+    /*
+  --width-card: 285px;
+  --width-card-medium: 460px;
+  --width-card-wide: 800px;
+*/
+    --width-content: 1080px;
+}
+
+@media (prefers-color-scheme: dark) {
+    :root[color-mode="user"] {
+        --color-accent: #0097fc4f;
+        --color-bg: #333;
+        --color-bg-secondary: #555;
+        --color-link: #0097fc;
+        --color-secondary: #e20de9;
+        --color-secondary-accent: #e20de94f;
+        --color-shadow: #bbbbbb20;
+        --color-table: #0097fc;
+        --color-text: #f7f7f7;
+        --color-text-secondary: #aaa;
+    }
+}

shared/Cargo.toml

@@ -1,10 +1,16 @@
 [package]
-name = "shared"
-version = "0.1.0"
-edition = "2021"
+name = "letterbox-shared"
+description = "Shared module for letterbox"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-notmuch = { path = "../notmuch" }
+build-info = "0.0.39"
+letterbox-notmuch = { version = "0.9.3", path = "../notmuch", registry = "xinu" }
 serde = { version = "1.0.147", features = ["derive"] }

shared/src/lib.rs

@@ -1,4 +1,7 @@
-use notmuch::SearchSummary;
+use std::hash::{DefaultHasher, Hash, Hasher};
+
+use build_info::{BuildInfo, VersionControl};
+use letterbox_notmuch::SearchSummary;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -12,3 +15,44 @@ pub struct SearchResult {
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct Message {}
+
+pub mod urls {
+    pub const MOUNT_POINT: &'static str = "/api";
+    pub fn cid_prefix(host: Option<&str>, cid: &str) -> String {
+        if let Some(host) = host {
+            format!("//{host}/api/cid/{cid}/")
+        } else {
+            format!("/api/cid/{cid}/")
+        }
+    }
+    pub fn download_attachment(host: Option<&str>, id: &str, idx: &str, filename: &str) -> String {
+        if let Some(host) = host {
+            format!(
+                "//{host}/api/download/attachment/{}/{}/{}",
+                id, idx, filename
+            )
+        } else {
+            format!("/api/download/attachment/{}/{}/{}", id, idx, filename)
+        }
+    }
+}
+pub fn build_version(bi: fn() -> &'static BuildInfo) -> String {
+    fn commit(git: &Option<VersionControl>) -> String {
+        let Some(VersionControl::Git(git)) = git else {
+            return String::new();
+        };
+        let mut s = vec!["-".to_string(), git.commit_short_id.clone()];
+        if let Some(branch) = &git.branch {
+            s.push(format!(" ({branch})"));
+        }
+        s.join("")
+    }
+    let bi = bi();
+
+    format!("v{}{}", bi.crate_info.version, commit(&bi.version_control)).to_string()
+}
+pub fn compute_color(data: &str) -> String {
+    let mut hasher = DefaultHasher::new();
+    data.hash(&mut hasher);
+    format!("#{:06x}", hasher.finish() % (1 << 24))
+}

web/Cargo.toml

@@ -1,16 +1,15 @@
 [package]
-version = "0.1.0"
-name = "letterbox"
-repository = "https://github.com/seed-rs/seed-quickstart"
-authors = ["Bill Thiede <git@xinu.tv>"]
-description = "App Description"
-categories = ["category"]
-license = "MIT"
-readme = "./README.md"
-edition = "2018"
+name = "letterbox-web"
+description = "Web frontend for letterbox"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
-[lib]
-crate-type = ["cdylib"]
+[build-dependencies]
+build-info-build = "0.0.39"
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.33"
@@ -18,20 +17,26 @@ wasm-bindgen-test = "0.3.33"
 [dependencies]
 console_error_panic_hook = "0.1.7"
 log = "0.4.17"
-seed = "0.9.2"
-console_log = {git = "http://git-private.h.xinu.tv/wathiede/console_log.git"}
+seed = { version = "0.10.0", features = ["routing"] }
+#seed = "0.9.2"
+console_log = { version = "0.1.0", registry = "xinu" }
 serde = { version = "1.0.147", features = ["derive"] }
-notmuch = {path = "../notmuch"}
-shared = {path = "../shared"}
-itertools = "0.10.5"
+itertools = "0.14.0"
 serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
-wasm-timer = "0.2.5"
-css-inline = "0.8.5"
 chrono = "0.4.31"
-graphql_client = "0.13.0"
-thiserror = "1.0.50"
-seed_hooks = { git = "https://github.com/seed-rs/styles_hooks", package = "seed_hooks", branch = "main" }
-
+graphql_client = "0.14.0"
+thiserror = "2.0.0"
+gloo-net = { version = "0.6.0", features = ["json", "serde_json"] }
+human_format = "1.1.0"
+build-info = "0.0.39"
+wasm-bindgen = "=0.2.100"
+uuid = { version = "1.13.1", features = [
+  "js",
+] } # direct dep to set js feature, prevents Rng issues
+letterbox-shared = { version = "0.9.3", path = "../shared", registry = "xinu" }
+letterbox-notmuch = { version = "0.9.3", path = "../notmuch", registry = "xinu" }
+seed_hooks = { version = "0.4.0", registry = "xinu" }
+strum_macros = "0.27.1"
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ['-Os']
@@ -39,6 +44,10 @@ wasm-opt = ['-Os']
 [dependencies.web-sys]
 version = "0.3.58"
 features = [
+  "Clipboard",
+  "DomRect",
+  "Element",
   "MediaQueryList",
-  "Window"
+  "Navigator",
+  "Window",
 ]

web/Makefile

@@ -1,7 +0,0 @@
-.PHONY: all
-
-# Build in release mode and push to minio for serving.
-all:
-	trunk build --release 
-	mc mirror  m/letterbox/ /tmp/letterbox-$(shell date +%s)
-	mc mirror --overwrite --remove dist/ m/letterbox/

web/Trunk.toml

@@ -1,5 +1,5 @@
 [build]
-release = true
+release = false
 
 [serve]
 # The address to serve on.
@@ -7,11 +7,14 @@ address = "0.0.0.0"
 port = 6758
 
 [[proxy]]
-backend = "http://localhost:9345/"
-rewrite= "/api/"
-[[proxy]]
-backend="http://localhost:9345/original"
-[[proxy]]
-backend="http://localhost:9345/graphiql"
-[[proxy]]
-backend="http://localhost:9345/graphql"
+backend = "http://localhost:9345/api/"
+
+[[hooks]]
+stage = "pre_build"
+command = "printf"
+command_arguments = ["\\033c"]
+
+#[[hooks]]
+#stage = "pre_build"
+#command = "cargo"
+#command_arguments = [ "test" ]

web/build.rs

@@ -0,0 +1,5 @@
+fn main() {
+    // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
+    // and `build_info::format!` in the main program.
+    build_info_build::build_script();
+}

web/graphql/add_tag.graphql

@@ -0,0 +1,3 @@
+mutation AddTagMutation($query: String!, $tag: String!) {
+  tagAdd(query:$query, tag:$tag) 
+}

web/graphql/catchup.graphql

@@ -0,0 +1,3 @@
+query CatchupQuery($query: String!) {
+    catchup(query: $query)
+}

web/graphql/front_page.graphql

@@ -9,10 +9,12 @@ query FrontPageQuery($query: String!, $after: String $before: String, $first: In
     }
     nodes {
       thread
+      total
       timestamp
       subject
       authors
       tags
+      corpus
     }
   }
   tags {
@@ -21,4 +23,5 @@ query FrontPageQuery($query: String!, $after: String $before: String, $first: In
     fgColor
     unread
   }
+  version
 }

web/graphql/mark_read.graphql

@@ -0,0 +1,3 @@
+mutation MarkReadMutation($query: String!, $unread: Boolean!) {
+  setReadStatus(query:$query, unread:$unread) 
+}

web/graphql/refresh.graphql

@@ -0,0 +1,3 @@
+mutation RefreshMutation {
+  refresh
+}

web/graphql/remove_tag.graphql

@@ -0,0 +1,3 @@
+mutation RemoveTagMutation($query: String!, $tag: String!) {
+  tagRemove(query:$query, tag:$tag) 
+}

web/graphql/show_thread.graphql

@@ -1,41 +1,69 @@
 query ShowThreadQuery($threadId: String!) {
   thread(threadId: $threadId) {
-    subject
-    messages {
-      id
-      subject
-      from {
-        name
-        addr
-      }
-      to {
-        name
-        addr
-      }
-      cc {
-        name
-        addr
-      }
+    __typename ... on NewsPost{
+      threadId
+      isRead
+      slug
+      site
+      title
+      body
+      url
       timestamp
-      body {
-        __typename
-        ... on UnhandledContentType {
-          contents
+      # TODO: unread
+    }
+    __typename ... on EmailThread{
+      threadId,
+      subject
+      messages {
+        id
+        subject
+        tags
+        from {
+          name
+          addr
+          photoUrl
         }
-        ... on PlainText {
-          contents
-          contentTree
+        to {
+          name
+          addr
         }
-        ... on Html {
-          contents
-          contentTree
+        cc {
+          name
+          addr
+        }
+        xOriginalTo {
+          name
+          addr
+        }
+        deliveredTo {
+          name
+          addr
+        }
+        timestamp
+        body {
+          __typename
+          ... on UnhandledContentType {
+            contents
+            contentTree
+          }
+          ... on PlainText {
+            contents
+            contentTree
+          }
+          ... on Html {
+            contents
+            contentTree
+          }
+        }
+        path
+        attachments {
+          id
+          idx
+          filename
+          contentType
+          contentId
+          size
         }
-      }
-      path
-      attachments {
-        filename
-        contentType
-        contentId
       }
     }
   }
@@ -45,4 +73,5 @@ query ShowThreadQuery($threadId: String!) {
     fgColor
     unread
   }
+  version
 }

web/graphql/update_schema.sh

@@ -1,4 +1,4 @@
 DEV_HOST=localhost
 DEV_PORT=9345
-graphql-client introspect-schema http://${DEV_HOST:?}:${DEV_PORT:?}/graphql --output schema.json
+graphql-client introspect-schema http://${DEV_HOST:?}:${DEV_PORT:?}/api/graphql --output schema.json
 git diff schema.json

web/index.html

@@ -2,167 +2,24 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <link rel="stylesheet", href="https://jenil.github.io/bulmaswatch/cyborg/bulmaswatch.min.css">
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.3.0/css/all.min.css" integrity="sha512-SzlrxWUlpfuzQ+pcUCosxcglQRNAq/DZjVsC0lE40xsADsfeQoEypE+enwcOiGjk/bSuGGKHEyjSoQ1zVisanQ==" crossorigin="anonymous" referrerpolicy="no-referrer" />
-    <link rel="icon" href="https://static.xinu.tv/favicon/letterbox.svg" />
-<style>
-
-.message {
-  padding: 0.5em;*/
-}
-.body {
-  background: white;
-  color: black;
-  padding: 0.5em;
-  margin-left: -0.5em;
-  margin-right: -0.5em;
-  margin-top: 0.5em;
-}
-.error {
-  background-color: red;
-}
-.view-part-text-plain {
-  white-space: pre-line;
-}
-iframe {
-  height: 100%;
-  width: 100%;
-}
-
-.index {
-  table-layout: fixed;
-  width: 100%;
-}
-.index .from {
-  overflow: hidden;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-  width: 10em;
-}
-.index .subject {
-  overflow: hidden;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-}
-.index .date {
-  width: 10em;
-  white-space: nowrap;
-  text-align: right;
-}
-.footer {
-  background-color: #eee;
-  color: #222;
-  position: fixed; 
-  bottom: 0;
-  left: 0;
-  right: 0;
-  height: 3em;
-  padding: 1em;
-}
-.tag {
-  margin-right: 2px;
-}
-.debug ul {
-  padding-left: 2em;
-}
-.debug li {
-}
-.loading {
-  animation-name: spin;
-  animation-duration: 1000ms;
-  animation-iteration-count: infinite;
-  animation-timing-function: linear;
-}
-@keyframes spin {
-  from {
-    transform:rotate(0deg);
-  }
-  to {
-    transform:rotate(360deg);
-  }
-}
-@media (max-width: 768px) {
-  .section {
-    padding: 1.5em;
-  }
-}
-input, .input {
-  color: #000;
-}
-input::placeholder, .input::placeholder{
-  color: #555;
-}
-
-.mobile .search-results,
-.mobile .thread  {
-  padding: 1em;
-}
-
-.search-results .row {
-  border-bottom: 1px #444 solid;
-  padding-bottom: .5em;
-  padding-top: .5em;
-}
-.search-results .row .subject {
-  overflow: hidden;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-}
-.search-results .row .from {
-  overflow: hidden;
-  text-overflow: ellipsis;
-  white-space: nowrap;
-}
-.search-results .row .tag {
-  height: 1.5em;
-  padding-left: .5em;
-  padding-right: .5em;
-}
-.float-right {
-  float: right;
-}
-/* Hide quoted emails */
-/*
-div[name="quote"],
-blockquote[type="cite"],
-.gmail_quote {
-  background-color: red;
-  display: none;
-}
-*/
-
-.desktop .main-content {
-  display: grid;
-  grid-template-columns: 12rem 1fr;
-}
-.tags-menu {
-  padding: 1rem;
-}
-.tags-menu .menu-list a {
-  padding: 0.25em 0.5em;
-}
-.tags-menu .tag-indent {
-  padding-left: .5em;
-}
-.tags-menu .tag-tag {
-  margin-left: -1em;
-  padding-right: .25em;
-}
-.navbar {
-  border: none;
-}
-.desktop nav.pagination,
-.tablet nav.pagination {
-  margin-left: .5em;
-  margin-bottom: 0 !important;
-}
-</style>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css"
+    integrity="sha512-Evv84Mr4kqVGRNSgIGL/F/aIDqQb7xQ2vcrdIwxfjThSH8CSR7PBEakCr51Ck+w+/U6swU2Im1vVX0SVk9ABhg=="
+    crossorigin="anonymous" referrerpolicy="no-referrer" />
+  <link rel="icon" href="https://static.xinu.tv/favicon/letterbox.svg" />
+  <!-- tall thin font for user icon -->
+  <link rel="preconnect" href="https://fonts.googleapis.com">
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+  <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@700&display=swap" rel="stylesheet">
+  <!-- <link data-trunk rel="css" href="static/site-specific.css" /> -->
+  <link data-trunk rel="css" href="static/vars.css" />
+  <link data-trunk rel="tailwind-css" href="./src/tailwind.css" />
+  <link data-trunk rel="css" href="static/overrides.css" />
 </head>
 
 <body>
-    <section id="app"></section>
+  <section id="app"></section>
 </body>
 
-</html>
+</html>

web/src/api.rs

@@ -1,21 +1,3 @@
-use log::info;
-use notmuch::ThreadSet;
-use seed::{prelude::*, Url};
-
-const BASE_URL: &str = "/api";
-pub fn refresh() -> String {
-    format!("{BASE_URL}/refresh")
-}
-pub fn search(query: &str, page: usize, results_per_page: usize) -> String {
-    let query = Url::encode_uri_component(query);
-    format!("{BASE_URL}/search/{query}?page={page}&results_per_page={results_per_page}")
-}
-pub fn show_pretty(tid: &str) -> String {
-    format!("{BASE_URL}/show/{tid}/pretty")
-}
-pub fn original(message_id: &str) -> String {
-    format!("{BASE_URL}/original/{message_id}")
-}
 pub mod urls {
     use seed::Url;
     pub fn search(query: &str, page: usize) -> Url {
@@ -30,39 +12,3 @@ pub mod urls {
         Url::new().set_hash_path(["t", tid])
     }
 }
-
-pub async fn search_request(
-    query: &str,
-    page: usize,
-    results_per_page: usize,
-) -> fetch::Result<shared::SearchResult> {
-    Request::new(search(query, page, results_per_page))
-        .method(Method::Get)
-        .fetch()
-        .await?
-        .check_status()?
-        .json()
-        .await
-}
-
-pub async fn refresh_request() -> fetch::Result<()> {
-    let t = Request::new(refresh())
-        .method(Method::Get)
-        .fetch()
-        .await?
-        .check_status()?
-        .text()
-        .await?;
-    info!("refresh {t}");
-    Ok(())
-}
-
-pub async fn show_pretty_request(tid: &str) -> fetch::Result<ThreadSet> {
-    Request::new(show_pretty(tid))
-        .method(Method::Get)
-        .fetch()
-        .await?
-        .check_status()?
-        .json()
-        .await
-}

web/src/consts.rs

@@ -1,2 +1 @@
 pub const SEARCH_RESULTS_PER_PAGE: usize = 20;
-pub const USE_GRAPHQL: bool = true;

web/src/graphql.rs

@@ -1,8 +1,5 @@
+use gloo_net::{http::Request, Error};
 use graphql_client::GraphQLQuery;
-use seed::{
-    fetch,
-    fetch::{Header, Method, Request},
-};
 use serde::{de::DeserializeOwned, Serialize};
 
 // The paths are relative to the directory where your `Cargo.toml` is located.
@@ -15,6 +12,14 @@ use serde::{de::DeserializeOwned, Serialize};
 )]
 pub struct FrontPageQuery;
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/catchup.graphql",
+    response_derives = "Debug"
+)]
+pub struct CatchupQuery;
+
 #[derive(GraphQLQuery)]
 #[graphql(
     schema_path = "graphql/schema.json",
@@ -23,21 +28,50 @@ pub struct FrontPageQuery;
 )]
 pub struct ShowThreadQuery;
 
-pub async fn send_graphql<Body, Resp>(body: Body) -> fetch::Result<graphql_client::Response<Resp>>
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/mark_read.graphql",
+    response_derives = "Debug"
+)]
+pub struct MarkReadMutation;
+
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/add_tag.graphql",
+    response_derives = "Debug"
+)]
+pub struct AddTagMutation;
+
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/remove_tag.graphql",
+    response_derives = "Debug"
+)]
+pub struct RemoveTagMutation;
+
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/refresh.graphql",
+    response_derives = "Debug"
+)]
+pub struct RefreshMutation;
+
+pub async fn send_graphql<Body, Resp>(body: Body) -> Result<graphql_client::Response<Resp>, Error>
 where
     Body: Serialize,
     Resp: DeserializeOwned + 'static,
 {
     use web_sys::RequestMode;
 
-    Request::new("/graphql/")
-        .method(Method::Post)
-        .header(Header::content_type("application/json"))
+    Request::post("/api/graphql/")
         .mode(RequestMode::Cors)
         .json(&body)?
-        .fetch()
+        .send()
         .await?
-        .check_status()?
         .json()
         .await
 }

web/src/main.rs

@@ -4,7 +4,7 @@
 #![allow(clippy::wildcard_imports)]
 
 use log::Level;
-use seed::prelude::*;
+use seed::App;
 
 mod api;
 mod consts;
@@ -12,9 +12,7 @@ mod graphql;
 mod state;
 mod view;
 
-// (This function is invoked by `init` function in `index.html`.)
-#[wasm_bindgen(start)]
-pub fn start() {
+fn main() {
     // This provides better error messages in debug mode.
     // It's disabled in release mode so it doesn't bloat up the file size.
     #[cfg(debug_assertions)]

web/src/model.rs

@@ -1 +0,0 @@
-

web/src/state.rs

@@ -1,121 +1,137 @@
+use std::collections::HashSet;
+
 use graphql_client::GraphQLQuery;
-use log::{debug, error, info};
-use notmuch::ThreadSet;
+use log::{debug, error, info, warn};
 use seed::{prelude::*, *};
 use thiserror::Error;
+use web_sys::HtmlElement;
 
 use crate::{
-    api,
     api::urls,
-    consts::{SEARCH_RESULTS_PER_PAGE, USE_GRAPHQL},
+    consts::SEARCH_RESULTS_PER_PAGE,
     graphql,
     graphql::{front_page_query::*, send_graphql, show_thread_query::*},
 };
 
+/// Used to fake the unread string while in development
+pub fn unread_query() -> &'static str {
+    let host = seed::window()
+        .location()
+        .host()
+        .expect("failed to get host");
+    if host.starts_with("6758.") {
+        return "tag:letterbox";
+    }
+    "is:unread"
+}
+
 // `init` describes what should happen when your app started.
 pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
+    let version = letterbox_shared::build_version(bi);
+    info!("Build Info: {}", version);
     if url.hash().is_none() {
-        orders.request_url(urls::search("is:unread", 0));
+        orders.request_url(urls::search(unread_query(), 0));
     } else {
         orders.notify(subs::UrlRequested::new(url));
     };
     orders.stream(streams::window_event(Ev::Resize, |_| Msg::OnResize));
+    // TODO(wathiede): only do this while viewing the index? Or maybe add a new message that force
+    // 'notmuch new' on the server periodically?
+    orders.stream(streams::interval(30_000, || Msg::RefreshStart));
     orders.subscribe(on_url_changed);
+    orders.stream(streams::window_event(Ev::Scroll, |_| Msg::WindowScrolled));
 
+    build_info::build_info!(fn bi);
     Model {
         context: Context::None,
         query: "".to_string(),
         refreshing_state: RefreshingState::None,
-        ui_error: UIError::NoError,
         tags: None,
+        read_completion_ratio: 0.,
+        content_el: ElRef::<HtmlElement>::default(),
+        versions: Version {
+            client: version,
+            server: None,
+        },
+        catchup: None,
     }
 }
 
 fn on_url_changed(uc: subs::UrlChanged) -> Msg {
     let mut url = uc.0;
+    let href = document().location().unwrap().href().unwrap();
+    let origin = document().location().unwrap().origin().unwrap();
+    let current_url = &href[origin.len()..];
+    let did_change = current_url != url.to_string();
+    let mut messages = Vec::new();
+    if did_change {
+        messages.push(Msg::ScrollToTop)
+    }
     info!(
-        "url changed '{}', history {}",
-        url,
+        "url changed\nold '{current_url}'\nnew '{url}', history {}",
         history().length().unwrap_or(0)
     );
     let hpp = url.remaining_hash_path_parts();
-    match hpp.as_slice() {
-        ["t", tid] => {
-            if USE_GRAPHQL {
-                Msg::ShowThreadRequest {
-                    thread_id: tid.to_string(),
-                }
-            } else {
-                Msg::ShowPrettyRequest(tid.to_string())
-            }
-        }
+    let msg = match hpp.as_slice() {
+        ["t", tid] => Msg::ShowThreadRequest {
+            thread_id: tid.to_string(),
+        },
         ["s", query] => {
             let query = Url::decode_uri_component(query).unwrap_or("".to_string());
-            if USE_GRAPHQL {
-                Msg::FrontPageRequest {
-                    query,
-                    after: None,
-                    before: None,
-                    first: None,
-                    last: None,
-                }
-            } else {
-                Msg::SearchRequest {
-                    query,
-                    page: 0,
-                    results_per_page: SEARCH_RESULTS_PER_PAGE,
-                }
+            Msg::FrontPageRequest {
+                query,
+                after: None,
+                before: None,
+                first: None,
+                last: None,
             }
         }
         ["s", query, page] => {
             let query = Url::decode_uri_component(query).unwrap_or("".to_string());
             let page = page[1..].parse().unwrap_or(0);
-            if USE_GRAPHQL {
-                Msg::FrontPageRequest {
-                    query,
-                    after: Some(page.to_string()),
-                    before: None,
-                    first: None,
-                    last: None,
-                }
-            } else {
-                Msg::SearchRequest {
-                    query,
-                    page,
-                    results_per_page: SEARCH_RESULTS_PER_PAGE,
-                }
+            Msg::FrontPageRequest {
+                query,
+                after: Some(page.to_string()),
+                before: None,
+                first: None,
+                last: None,
             }
         }
         p => {
             if !p.is_empty() {
                 info!("Unhandled path '{p:?}'");
             }
-            if USE_GRAPHQL {
-                Msg::FrontPageRequest {
-                    query: "".to_string(),
-                    after: None,
-                    before: None,
-                    first: None,
-                    last: None,
-                }
-            } else {
-                Msg::SearchRequest {
-                    query: "".to_string(),
-                    page: 0,
-                    results_per_page: SEARCH_RESULTS_PER_PAGE,
-                }
+            Msg::FrontPageRequest {
+                query: "".to_string(),
+                after: None,
+                before: None,
+                first: None,
+                last: None,
             }
         }
-    }
+    };
+    messages.push(msg);
+    Msg::MultiMsg(messages)
 }
 
 // `update` describes how to handle each `Msg`.
 pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
+    debug!("update({})", msg);
     match msg {
         Msg::Noop => {}
         Msg::RefreshStart => {
             model.refreshing_state = RefreshingState::Loading;
-            orders.perform_cmd(async move { Msg::RefreshDone(api::refresh_request().await.err()) });
+            orders.perform_cmd(async move {
+                Msg::RefreshDone(
+                    send_graphql::<_, graphql::refresh_mutation::ResponseData>(
+                        graphql::RefreshMutation::build_query(
+                            graphql::refresh_mutation::Variables {},
+                        ),
+                    )
+                    .await
+                    .err(),
+                )
+            });
         }
         Msg::RefreshDone(err) => {
             model.refreshing_state = if let Some(err) = err {
@@ -123,49 +139,21 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             } else {
                 RefreshingState::None
             };
-            orders.perform_cmd(async move { Msg::Reload });
+            orders.perform_cmd(async move { Msg::Refresh });
+        }
+        Msg::Refresh => {
+            orders.perform_cmd(async move { on_url_changed(subs::UrlChanged(Url::current())) });
         }
         Msg::Reload => {
-            orders.perform_cmd(async move { on_url_changed(subs::UrlChanged(Url::current())) });
+            window()
+                .location()
+                .reload()
+                .expect("failed to reload window");
         }
         Msg::OnResize => (),
 
-        Msg::SearchRequest {
-            query,
-            page,
-            results_per_page,
-        } => {
-            info!("searching for '{query}' pg {page} # / pg {results_per_page}");
-            model.query = query.clone();
-            orders.skip().perform_cmd(async move {
-                Msg::SearchResult(api::search_request(&query, page, results_per_page).await)
-            });
-        }
-        Msg::SearchResult(Ok(response_data)) => {
-            debug!("fetch ok {:#?}", response_data);
-            model.context = Context::Search(response_data);
-        }
-        Msg::SearchResult(Err(fetch_error)) => {
-            error!("fetch failed {:?}", fetch_error);
-        }
-
-        Msg::ShowPrettyRequest(tid) => {
-            orders.skip().perform_cmd(async move {
-                Msg::ShowPrettyResult(api::show_pretty_request(&tid).await)
-            });
-        }
-        Msg::ShowPrettyResult(Ok(response_data)) => {
-            debug!("fetch ok {:#?}", response_data);
-            model.context = Context::Thread(response_data);
-        }
-        Msg::ShowPrettyResult(Err(fetch_error)) => {
-            error!("fetch failed {:?}", fetch_error);
-        }
         Msg::NextPage => {
             match &model.context {
-                Context::Search(sr) => {
-                    orders.request_url(urls::search(&sr.query, sr.page + 1));
-                }
                 Context::SearchResult { query, pager, .. } => {
                     let query = query.to_string();
                     let after = pager.end_cursor.clone();
@@ -179,16 +167,12 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                         }
                     });
                 }
-                Context::Thread(_) => (),       // do nothing (yet?)
-                Context::ThreadResult(_) => (), // do nothing (yet?)
-                Context::None => (),            // do nothing (yet?)
+                Context::ThreadResult { .. } => (), // do nothing (yet?)
+                Context::None => (),                // do nothing (yet?)
             };
         }
         Msg::PreviousPage => {
             match &model.context {
-                Context::Search(sr) => {
-                    orders.request_url(urls::search(&sr.query, sr.page.saturating_sub(1)));
-                }
                 Context::SearchResult { query, pager, .. } => {
                     let query = query.to_string();
                     let before = pager.start_cursor.clone();
@@ -203,17 +187,75 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     });
                 }
 
-                Context::Thread(_) => (),       // do nothing (yet?)
-                Context::ThreadResult(_) => (), // do nothing (yet?)
-                Context::None => (),            // do nothing (yet?)
+                Context::ThreadResult { .. } => (), // do nothing (yet?)
+                Context::None => (),                // do nothing (yet?)
             };
         }
+        Msg::GoToSearchResults => {
+            orders.send_msg(Msg::SearchQuery(model.query.clone()));
+        }
 
         Msg::UpdateQuery(query) => model.query = query,
         Msg::SearchQuery(query) => {
             orders.request_url(urls::search(&query, 0));
         }
 
+        Msg::SetUnread(query, unread) => {
+            orders.skip().perform_cmd(async move {
+                let res: Result<
+                    graphql_client::Response<graphql::mark_read_mutation::ResponseData>,
+                    gloo_net::Error,
+                > = send_graphql(graphql::MarkReadMutation::build_query(
+                    graphql::mark_read_mutation::Variables {
+                        query: query.clone(),
+                        unread,
+                    },
+                ))
+                .await;
+                if let Err(e) = res {
+                    error!("Failed to set read for {query} to {unread}: {e}");
+                }
+                Msg::Refresh
+            });
+        }
+        Msg::AddTag(query, tag) => {
+            orders.skip().perform_cmd(async move {
+                let res: Result<
+                    graphql_client::Response<graphql::add_tag_mutation::ResponseData>,
+                    gloo_net::Error,
+                > = send_graphql(graphql::AddTagMutation::build_query(
+                    graphql::add_tag_mutation::Variables {
+                        query: query.clone(),
+                        tag: tag.clone(),
+                    },
+                ))
+                .await;
+                if let Err(e) = res {
+                    error!("Failed to add tag {tag} to {query}: {e}");
+                }
+                Msg::GoToSearchResults
+            });
+        }
+        Msg::RemoveTag(query, tag) => {
+            orders.skip().perform_cmd(async move {
+                let res: Result<
+                    graphql_client::Response<graphql::remove_tag_mutation::ResponseData>,
+                    gloo_net::Error,
+                > = send_graphql(graphql::RemoveTagMutation::build_query(
+                    graphql::remove_tag_mutation::Variables {
+                        query: query.clone(),
+                        tag: tag.clone(),
+                    },
+                ))
+                .await;
+                if let Err(e) = res {
+                    error!("Failed to remove tag {tag} to {query}: {e}");
+                }
+                // TODO: reconsider this behavior
+                Msg::GoToSearchResults
+            });
+        }
+
         Msg::FrontPageRequest {
             query,
             after,
@@ -221,7 +263,14 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             first,
             last,
         } => {
-            info!("making FrontPageRequest: {query} after:{after:?} before:{before:?} first:{first:?} last:{last:?}");
+            let (after, before, first, last) = match (after.as_ref(), before.as_ref(), first, last)
+            {
+                // If no pagination set, set reasonable defaults
+                (None, None, None, None) => {
+                    (None, None, Some(SEARCH_RESULTS_PER_PAGE as i64), None)
+                }
+                _ => (after, before, first, last),
+            };
             model.query = query.clone();
             orders.skip().perform_cmd(async move {
                 Msg::FrontPageResult(
@@ -262,17 +311,36 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .map(|t| Tag {
                         name: t.name,
                         bg_color: t.bg_color,
-                        fg_color: t.fg_color,
                         unread: t.unread,
                     })
                     .collect(),
             );
+            let selected_threads = 'context: {
+                if let Context::SearchResult {
+                    results,
+                    selected_threads,
+                    ..
+                } = &model.context
+                {
+                    let old: HashSet<_> = results.iter().map(|n| &n.thread).collect();
+                    let new: HashSet<_> = data.search.nodes.iter().map(|n| &n.thread).collect();
+
+                    if old == new {
+                        break 'context selected_threads.clone();
+                    }
+                }
+                HashSet::new()
+            };
             model.context = Context::SearchResult {
                 query: model.query.clone(),
                 results: data.search.nodes,
                 count: data.count as usize,
                 pager: data.search.page_info,
+                selected_threads,
             };
+            orders.send_msg(Msg::UpdateServerVersion(data.version));
+            // Generate signal so progress bar is reset
+            orders.send_msg(Msg::WindowScrolled);
         }
 
         Msg::ShowThreadRequest { thread_id } => {
@@ -294,25 +362,326 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .map(|t| Tag {
                         name: t.name,
                         bg_color: t.bg_color,
-                        fg_color: t.fg_color,
                         unread: t.unread,
                     })
                     .collect(),
             );
-            model.context = Context::ThreadResult(data.thread);
+            match &data.thread {
+                graphql::show_thread_query::ShowThreadQueryThread::EmailThread(
+                    ShowThreadQueryThreadOnEmailThread { messages, .. },
+                ) => {
+                    let mut open_messages: HashSet<_> = messages
+                        .iter()
+                        .filter(|msg| msg.tags.iter().any(|t| t == "unread"))
+                        .map(|msg| msg.id.clone())
+                        .collect();
+                    if open_messages.is_empty() {
+                        open_messages = messages.iter().map(|msg| msg.id.clone()).collect();
+                    }
+                    model.context = Context::ThreadResult {
+                        thread: data.thread,
+                        open_messages,
+                    };
+                }
+                graphql::show_thread_query::ShowThreadQueryThread::NewsPost(..) => {
+                    model.context = Context::ThreadResult {
+                        thread: data.thread,
+                        open_messages: HashSet::new(),
+                    };
+                }
+            }
+            orders.send_msg(Msg::UpdateServerVersion(data.version));
+            // Generate signal so progress bar is reset
+            orders.send_msg(Msg::WindowScrolled);
         }
         Msg::ShowThreadResult(bad) => {
             error!("show_thread_query error: {bad:#?}");
         }
+        Msg::CatchupRequest { query } => {
+            orders.perform_cmd(async move {
+                Msg::CatchupResult(
+                    send_graphql::<_, graphql::catchup_query::ResponseData>(
+                        graphql::CatchupQuery::build_query(graphql::catchup_query::Variables {
+                            query,
+                        }),
+                    )
+                    .await,
+                )
+            });
+        }
+        Msg::CatchupResult(Ok(graphql_client::Response {
+            data: Some(data), ..
+        })) => {
+            let items = data.catchup;
+            if items.is_empty() {
+                orders.send_msg(Msg::GoToSearchResults);
+                model.catchup = None;
+            } else {
+                orders.request_url(urls::thread(&items[0]));
+                model.catchup = Some(Catchup {
+                    items: items
+                        .into_iter()
+                        .map(|id| CatchupItem { id, seen: false })
+                        .collect(),
+                });
+            }
+        }
+        Msg::CatchupResult(bad) => {
+            error!("catchup_query error: {bad:#?}");
+        }
+        Msg::SelectionSetNone => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                *selected_threads = HashSet::new();
+            }
+        }
+        Msg::SelectionSetAll => {
+            if let Context::SearchResult {
+                results,
+                selected_threads,
+                ..
+            } = &mut model.context
+            {
+                *selected_threads = results.iter().map(|node| node.thread.clone()).collect();
+            }
+        }
+        Msg::SelectionAddTag(tag) => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                let threads = selected_threads
+                    .iter()
+                    .map(|tid| tid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                orders
+                    .skip()
+                    .perform_cmd(async move { Msg::AddTag(threads, tag) });
+            }
+        }
+        Msg::SelectionRemoveTag(tag) => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                let threads = selected_threads
+                    .iter()
+                    .map(|tid| tid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                orders
+                    .skip()
+                    .perform_cmd(async move { Msg::RemoveTag(threads, tag) });
+            }
+        }
+        Msg::SelectionMarkAsRead => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                let threads = selected_threads
+                    .iter()
+                    .map(|tid| tid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                orders
+                    .skip()
+                    .perform_cmd(async move { Msg::SetUnread(threads, false) });
+            }
+        }
+        Msg::SelectionMarkAsUnread => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                let threads = selected_threads
+                    .iter()
+                    .map(|tid| tid.to_string())
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                orders
+                    .skip()
+                    .perform_cmd(async move { Msg::SetUnread(threads, true) });
+            }
+        }
+        Msg::SelectionAddThread(tid) => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                selected_threads.insert(tid);
+            }
+        }
+        Msg::SelectionRemoveThread(tid) => {
+            if let Context::SearchResult {
+                selected_threads, ..
+            } = &mut model.context
+            {
+                selected_threads.remove(&tid);
+            }
+        }
+        Msg::MessageCollapse(id) => {
+            if let Context::ThreadResult { open_messages, .. } = &mut model.context {
+                open_messages.remove(&id);
+            }
+        }
+        Msg::MessageExpand(id) => {
+            if let Context::ThreadResult { open_messages, .. } = &mut model.context {
+                open_messages.insert(id);
+            }
+        }
+        Msg::MultiMsg(msgs) => msgs.into_iter().for_each(|msg| update(msg, model, orders)),
+        Msg::CopyToClipboard(text) => {
+            let clipboard = seed::window().navigator().clipboard();
+            orders.perform_cmd(async move {
+                wasm_bindgen_futures::JsFuture::from(clipboard.write_text(&text))
+                    .await
+                    .expect("failed to copy to clipboard");
+            });
+        }
+        Msg::ScrollToTop => {
+            info!("scrolling to the top");
+            web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
+        }
+        Msg::WindowScrolled => {
+            if let Some(el) = model.content_el.get() {
+                let ih = window()
+                    .inner_height()
+                    .expect("window height")
+                    .unchecked_into::<js_sys::Number>()
+                    .value_of();
+
+                let r = el.get_bounding_client_rect();
+                if r.height() < ih {
+                    // The whole content fits in the window, no scrollbar
+                    orders.send_msg(Msg::SetProgress(0.));
+                    return;
+                }
+                let end: f64 = r.height() - ih;
+                if end < 0. {
+                    orders.send_msg(Msg::SetProgress(0.));
+                    return;
+                }
+                // Flip Y, normally it's 0-point when the top of the content hits the top of the
+                // screen and goes negative from there.
+                let y = -r.y();
+                let ratio: f64 = (y / end).max(0.);
+                debug!(
+                    "WindowScrolled ih {ih} end {end} ratio {ratio:.02} {}x{} @ {},{}",
+                    r.width(),
+                    r.height(),
+                    r.x(),
+                    r.y()
+                );
+
+                orders.send_msg(Msg::SetProgress(ratio));
+            } else {
+                orders.send_msg(Msg::SetProgress(0.));
+            }
+        }
+        Msg::SetProgress(ratio) => {
+            model.read_completion_ratio = ratio;
+        }
+        Msg::UpdateServerVersion(version) => {
+            if version != model.versions.client {
+                warn!(
+                    "Server ({}) and client ({}) version mismatch, reloading",
+                    version, model.versions.client
+                );
+                orders.send_msg(Msg::Reload);
+            }
+            model.versions.server = Some(version);
+        }
+
+        Msg::CatchupStart => {
+            let query = if model.query.contains("is:unread") {
+                model.query.to_string()
+            } else {
+                format!("{} is:unread", model.query)
+            };
+            info!("starting catchup mode w/ {}", query);
+            orders.send_msg(Msg::ScrollToTop);
+            orders.send_msg(Msg::CatchupRequest { query });
+        }
+        Msg::CatchupKeepUnread => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, true));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupMarkAsRead => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, false));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupNext => {
+            orders.send_msg(Msg::ScrollToTop);
+            let Some(catchup) = &mut model.catchup else {
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            let Some(idx) = catchup.items.iter().position(|i| !i.seen) else {
+                // All items have been seen
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            catchup.items[idx].seen = true;
+            if idx < catchup.items.len() - 1 {
+                // Reached last item
+                orders.request_url(urls::thread(&catchup.items[idx + 1].id));
+                return;
+            } else {
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+        }
+        Msg::CatchupExit => {
+            orders.send_msg(Msg::ScrollToTop);
+            model.catchup = None;
+        }
     }
 }
+
+fn current_thread_id(context: &Context) -> Option<String> {
+    match context {
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                    thread_id, ..
+                }),
+            ..
+        } => Some(thread_id.clone()),
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::NewsPost(ShowThreadQueryThreadOnNewsPost { thread_id, .. }),
+            ..
+        } => Some(thread_id.clone()),
+        _ => None,
+    }
+}
+
 // `Model` describes our app state.
 pub struct Model {
     pub query: String,
     pub context: Context,
     pub refreshing_state: RefreshingState,
-    pub ui_error: UIError,
     pub tags: Option<Vec<Tag>>,
+    pub read_completion_ratio: f64,
+    pub content_el: ElRef<HtmlElement>,
+    pub versions: Version,
+    pub catchup: Option<Catchup>,
+}
+
+#[derive(Debug)]
+pub struct Version {
+    pub client: String,
+    pub server: Option<String>,
 }
 
 #[derive(Error, Debug)]
@@ -321,7 +690,7 @@ pub enum UIError {
     #[error("No error, this should never be presented to user")]
     NoError,
     #[error("failed to fetch {0}: {1:?}")]
-    FetchError(&'static str, FetchError),
+    FetchError(&'static str, gloo_net::Error),
     #[error("{0} error decoding: {1:?}")]
     FetchDecodeError(&'static str, Vec<graphql_client::Error>),
     #[error("no data or errors for {0}")]
@@ -330,21 +699,31 @@ pub enum UIError {
 
 pub enum Context {
     None,
-    Search(shared::SearchResult),
     SearchResult {
         query: String,
         results: Vec<FrontPageQuerySearchNodes>,
         count: usize,
         pager: FrontPageQuerySearchPageInfo,
+        selected_threads: HashSet<String>,
     },
-    Thread(ThreadSet),
-    ThreadResult(ShowThreadQueryThread),
+    ThreadResult {
+        thread: ShowThreadQueryThread,
+        open_messages: HashSet<String>,
+    },
+}
+
+pub struct Catchup {
+    pub items: Vec<CatchupItem>,
+}
+
+pub struct CatchupItem {
+    pub id: String,
+    pub seen: bool,
 }
 
 pub struct Tag {
     pub name: String,
     pub bg_color: String,
-    pub fg_color: String,
     pub unread: i64,
 }
 
@@ -355,28 +734,28 @@ pub enum RefreshingState {
     Error(String),
 }
 // `Msg` describes the different events you can modify state with.
+#[derive(strum_macros::Display)]
 pub enum Msg {
     Noop,
     // Tell the client to refresh its state
+    Refresh,
+    // Tell the client to reload whole page from server
     Reload,
     // Window has changed size
     OnResize,
     // Tell the server to update state
     RefreshStart,
-    RefreshDone(Option<FetchError>),
-    SearchRequest {
-        query: String,
-        page: usize,
-        results_per_page: usize,
-    },
-    SearchResult(fetch::Result<shared::SearchResult>),
-    ShowPrettyRequest(String),
-    ShowPrettyResult(fetch::Result<ThreadSet>),
+    RefreshDone(Option<gloo_net::Error>),
     NextPage,
     PreviousPage,
+    GoToSearchResults,
     UpdateQuery(String),
     SearchQuery(String),
 
+    SetUnread(String, bool),
+    AddTag(String, String),
+    RemoveTag(String, String),
+
     FrontPageRequest {
         query: String,
         after: Option<String>,
@@ -385,12 +764,45 @@ pub enum Msg {
         last: Option<i64>,
     },
     FrontPageResult(
-        fetch::Result<graphql_client::Response<graphql::front_page_query::ResponseData>>,
+        Result<graphql_client::Response<graphql::front_page_query::ResponseData>, gloo_net::Error>,
     ),
     ShowThreadRequest {
         thread_id: String,
     },
     ShowThreadResult(
-        fetch::Result<graphql_client::Response<graphql::show_thread_query::ResponseData>>,
+        Result<graphql_client::Response<graphql::show_thread_query::ResponseData>, gloo_net::Error>,
     ),
+    CatchupRequest {
+        query: String,
+    },
+    CatchupResult(
+        Result<graphql_client::Response<graphql::catchup_query::ResponseData>, gloo_net::Error>,
+    ),
+
+    SelectionSetNone,
+    SelectionSetAll,
+    SelectionAddTag(String),
+    #[allow(dead_code)]
+    SelectionRemoveTag(String),
+    SelectionMarkAsRead,
+    SelectionMarkAsUnread,
+    SelectionAddThread(String),
+    SelectionRemoveThread(String),
+
+    MessageCollapse(String),
+    MessageExpand(String),
+    MultiMsg(Vec<Msg>),
+
+    CopyToClipboard(String),
+
+    ScrollToTop,
+    WindowScrolled,
+    SetProgress(f64),
+    UpdateServerVersion(String),
+
+    CatchupStart,
+    CatchupKeepUnread,
+    CatchupMarkAsRead,
+    CatchupNext,
+    CatchupExit,
 }

web/src/tailwind.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

web/src/view/desktop.rs

@@ -1,121 +0,0 @@
-use log::info;
-use seed::{prelude::*, *};
-use seed_hooks::{state_access::CloneState, topo, use_state};
-
-use crate::{
-    api::urls,
-    state::{Context, Model, Msg, Tag},
-    view::{self, legacy, view_header, view_search_results},
-};
-
-#[topo::nested]
-pub(super) fn view(model: &Model) -> Node<Msg> {
-    // Do two queries, one without `unread` so it loads fast, then a second with unread.
-    let content = match &model.context {
-        Context::None => div![h1!["Loading"]],
-        Context::Thread(thread_set) => legacy::thread(thread_set),
-        Context::ThreadResult(thread) => view::thread(thread),
-        Context::Search(search_results) => legacy::search_results(&model.query, search_results),
-        Context::SearchResult {
-            query,
-            results,
-            count,
-            pager,
-        } => view_search_results(&query, results.as_slice(), *count, pager),
-    };
-    fn view_tag_li(display_name: &str, indent: usize, t: &Tag) -> Node<Msg> {
-        li![a![
-            attrs! {
-                At::Href => urls::search(&format!("tag:{}", t.name), 0)
-            },
-            (0..indent).map(|_| span![C!["tag-indent"], ""]),
-            i![
-                C!["tag-tag", "fa-solid", "fa-tag"],
-                style! {
-                    //"--fa-primary-color" => t.fg_color,
-                    St::Color => t.bg_color,
-                },
-            ],
-            display_name,
-            IF!(t.unread>0 => format!(" ({})", t.unread))
-        ]]
-    }
-    fn matches(a: &[&str], b: &[&str]) -> usize {
-        std::iter::zip(a.iter(), b.iter())
-            .take_while(|(a, b)| a == b)
-            .count()
-    }
-    fn view_tag_list<'a>(tags: impl Iterator<Item = &'a Tag>) -> Vec<Node<Msg>> {
-        let mut lis = Vec::new();
-        let mut last = Vec::new();
-        for t in tags {
-            let parts: Vec<_> = t.name.split('/').collect();
-            let mut n = matches(&last, &parts);
-            if t.name.starts_with("ZZCrap/Free") {
-                info!("n: {n}, parts: {parts:?} last: {last:?}");
-            }
-            if n <= parts.len() - 2 && parts.len() > 1 {
-                // Synthesize fake tags for proper indenting.
-                for i in n..parts.len() - 1 {
-                    let display_name = parts[n];
-                    lis.push(view_tag_li(
-                        &display_name,
-                        n,
-                        &Tag {
-                            name: parts[..i + 1].join("/"),
-                            bg_color: "#fff".to_string(),
-                            fg_color: "#000".to_string(),
-                            unread: 0,
-                        },
-                    ));
-                }
-                last = parts[..parts.len() - 1].to_vec();
-                n = parts.len() - 1;
-            }
-            let display_name = parts[n];
-            lis.push(view_tag_li(&display_name, n, t));
-            last = parts;
-        }
-        lis
-    }
-    let unread = model
-        .tags
-        .as_ref()
-        .map(|tags| tags.iter().filter(|t| t.unread > 0).collect())
-        .unwrap_or(Vec::new());
-    let tags_open = use_state(|| false);
-    let force_tags_open = unread.is_empty();
-    div![
-        C!["main-content"],
-        aside![
-            C!["tags-menu", "menu"],
-            IF!(!unread.is_empty() => p![C!["menu-label"], "Unread"]),
-            IF!(!unread.is_empty() => ul![C!["menu-list"], view_tag_list(unread.into_iter())]),
-            p![
-                C!["menu-label"],
-                IF!(!force_tags_open =>
-                i![C![
-                    "fa-solid",
-                    if tags_open.get() {
-                        "fa-angle-up"
-                    } else {
-                        "fa-angle-down"
-                    }
-                ]]),
-                " Tags",
-                ev(Ev::Click, move |_| {
-                    tags_open.set(!tags_open.get());
-                })
-            ],
-            ul![
-                C!["menu-list"],
-                IF!(force_tags_open||tags_open.get() => model.tags.as_ref().map(|tags| view_tag_list(tags.iter()))),
-            ]
-        ],
-        div![
-            view_header(&model.query, &model.refreshing_state),
-            content,
-            view_header(&model.query, &model.refreshing_state),
-        ]
-    ]
-}

web/src/view/legacy.rs

@@ -1,266 +0,0 @@
-use notmuch::{Content, Part, ThreadNode, ThreadSet};
-use seed::{prelude::*, *};
-
-use crate::{
-    api,
-    api::urls,
-    consts::SEARCH_RESULTS_PER_PAGE,
-    state::Msg,
-    view::{human_age, pretty_authors, set_title, tags_chiclet},
-};
-
-pub(super) fn search_results(query: &str, search_results: &shared::SearchResult) -> Node<Msg> {
-    if query.is_empty() {
-        set_title("all mail");
-    } else {
-        set_title(query);
-    }
-    let summaries = &search_results.summary.0;
-    let rows = summaries.iter().map(|r| {
-        let tid = r.thread.clone();
-        let datetime = human_age(r.timestamp as i64);
-        tr![
-            td![
-                C!["from"],
-                pretty_authors(&r.authors),
-                IF!(r.total>1 => small![" ", r.total.to_string()]),
-            ],
-            td![
-                C!["subject"],
-                tags_chiclet(&r.tags, false),
-                " ",
-                a![
-                    C!["has-text-light"],
-                    attrs! {
-                        At::Href => urls::thread(&tid)
-                    },
-                    &r.subject,
-                ]
-            ],
-            td![C!["date"], datetime]
-        ]
-    });
-    let first = search_results.page * search_results.results_per_page;
-    div![
-        search_pager(first, summaries.len(), search_results.total),
-        table![
-            C![
-                "table",
-                "index",
-                "is-fullwidth",
-                "is-hoverable",
-                "is-narrow",
-                "is-striped",
-            ],
-            thead![tr![
-                th![C!["from"], "From"],
-                th![C!["subject"], "Subject"],
-                th![C!["date"], "Date"]
-            ]],
-            tbody![rows]
-        ],
-        search_pager(first, summaries.len(), search_results.total)
-    ]
-}
-
-fn search_pager(start: usize, count: usize, total: usize) -> Node<Msg> {
-    let is_first = start <= 0;
-    let is_last = (start + SEARCH_RESULTS_PER_PAGE) >= total;
-    nav![
-        C!["pagination"],
-        a![
-            C!["pagination-previous", "button",],
-            IF!(is_first => attrs!{ At::Disabled=>true }),
-            "<",
-            ev(Ev::Click, |_| Msg::PreviousPage)
-        ],
-        a![
-            C!["pagination-next", "button", IF!(is_last => "is-static")],
-            IF!(is_last => attrs!{ At::Disabled=>true }),
-            ">",
-            ev(Ev::Click, |_| Msg::NextPage)
-        ],
-        ul![
-            C!["pagination-list"],
-            li![format!("{} - {} of {}", start, start + count, total)],
-        ],
-    ]
-}
-
-pub(super) fn thread(thread_set: &ThreadSet) -> Node<Msg> {
-    assert_eq!(thread_set.0.len(), 1);
-    let thread = &thread_set.0[0];
-    assert_eq!(thread.0.len(), 1);
-    let thread_node = &thread.0[0];
-    let subject = first_subject(&thread_node).unwrap_or("<No subject>".to_string());
-    set_title(&subject);
-    div![
-        C!["container"],
-        h1![C!["title"], subject],
-        view_message(&thread_node),
-        a![
-            attrs! {At::Href=>api::original(&thread_node.0.as_ref().expect("message missing").id)},
-            "Original"
-        ],
-    ]
-}
-pub(super) fn mobile_search_results(
-    query: &str,
-    search_results: &shared::SearchResult,
-) -> Node<Msg> {
-    if query.is_empty() {
-        set_title("all mail");
-    } else {
-        set_title(query);
-    }
-    let summaries = &search_results.summary.0;
-    let rows = summaries.iter().map(|r| {
-        /*
-        let tid = r.thread.clone();
-        tr![
-        td![
-        C!["from"],
-        pretty_authors(&r.authors),
-        IF!(r.total>1 => small![" ", r.total.to_string()]),
-        ],
-        td![C!["subject"], tags_chiclet(&r.tags), " ", &r.subject],
-        td![C!["date"], &r.date_relative],
-        ev(Ev::Click, move |_| Msg::ShowPrettyRequest(tid)),
-        ]
-        */
-        let tid = r.thread.clone();
-        let datetime = human_age(r.timestamp as i64);
-        a![
-            C!["has-text-light"],
-            attrs! {
-                At::Href => urls::thread(&tid)
-            },
-            div![
-                C!["row"],
-                div![C!["subject"], &r.subject],
-                span![C!["from", "is-size-7"], pretty_authors(&r.authors)],
-                div![
-                    span![C!["is-size-7"], tags_chiclet(&r.tags, true)],
-                    span![C!["is-size-7", "float-right", "date"], datetime]
-                ]
-            ]
-        ]
-    });
-    let first = search_results.page * search_results.results_per_page;
-    div![
-        C!["search-results"],
-        search_pager(first, summaries.len(), search_results.total),
-        rows,
-        search_pager(first, summaries.len(), search_results.total)
-    ]
-}
-fn view_message(thread: &ThreadNode) -> Node<Msg> {
-    let message = thread.0.as_ref().expect("ThreadNode missing Message");
-    let children = &thread.1;
-    div![
-        C!["message"],
-        /* TODO(wathiede): collect all the tags and show them here. */
-        /* TODO(wathiede): collect all the attachments from all the subparts */
-        div![C!["header"], "From: ", &message.headers.from],
-        div![C!["header"], "Date: ", &message.headers.date],
-        div![C!["header"], "To: ", &message.headers.to],
-        div![
-            C!["body"],
-            match &message.body {
-                Some(body) => view_body(body.as_slice()),
-                None => div!["<no body>"],
-            },
-        ],
-        children.iter().map(view_message)
-    ]
-}
-
-fn view_body(body: &[Part]) -> Node<Msg> {
-    div![body.iter().map(view_part)]
-}
-
-fn view_text_plain(content: &Option<Content>) -> Node<Msg> {
-    match &content {
-        Some(Content::String(content)) => p![C!["view-part-text-plain"], content],
-        _ => div![
-            C!["error"],
-            format!("Unhandled content enum for text/plain"),
-        ],
-    }
-}
-
-fn view_part(part: &Part) -> Node<Msg> {
-    match part.content_type.as_str() {
-        "text/plain" => view_text_plain(&part.content),
-        "text/html" => {
-            if let Some(Content::String(html)) = &part.content {
-                let inliner = css_inline::CSSInliner::options()
-                    .load_remote_stylesheets(false)
-                    .remove_style_tags(true)
-                    .build();
-                let inlined = inliner.inline(html).expect("failed to inline CSS");
-
-                return div![C!["view-part-text-html"], raw![&inlined]];
-            } else {
-                div![
-                    C!["error"],
-                    format!("Unhandled content enum for multipart/mixed"),
-                ]
-            }
-        }
-
-        // https://en.wikipedia.org/wiki/MIME#alternative
-        // RFC1341 states: In general, user agents that compose multipart/alternative entities
-        // should place the body parts in increasing order of preference, that is, with the
-        // preferred format last.
-        "multipart/alternative" => {
-            if let Some(Content::Multipart(parts)) = &part.content {
-                for part in parts.iter().rev() {
-                    if part.content_type == "text/html" {
-                        if let Some(Content::String(html)) = &part.content {
-                            let inliner = css_inline::CSSInliner::options()
-                                .load_remote_stylesheets(false)
-                                .remove_style_tags(true)
-                                .build();
-                            let inlined = inliner.inline(html).expect("failed to inline CSS");
-                            return div![Node::from_html(None, &inlined)];
-                        }
-                    }
-                    if part.content_type == "text/plain" {
-                        return view_text_plain(&part.content);
-                    }
-                }
-                div!["No known  multipart/alternative parts"]
-            } else {
-                div![
-                    C!["error"],
-                    format!("multipart/alternative with non-multipart content"),
-                ]
-            }
-        }
-        "multipart/mixed" => match &part.content {
-            Some(Content::Multipart(parts)) => div![parts.iter().map(view_part)],
-            _ => div![
-                C!["error"],
-                format!("Unhandled content enum for multipart/mixed"),
-            ],
-        },
-        _ => div![
-            C!["error"],
-            format!("Unhandled content type: {}", part.content_type)
-        ],
-    }
-}
-
-fn first_subject(thread: &ThreadNode) -> Option<String> {
-    if let Some(msg) = &thread.0 {
-        return Some(msg.headers.subject.clone());
-    } else {
-        for tn in &thread.1 {
-            if let Some(s) = first_subject(&tn) {
-                return Some(s);
-            }
-        }
-    }
-    None
-}

web/src/view/mobile.rs

@@ -1,71 +0,0 @@
-use seed::{prelude::*, *};
-
-use crate::{
-    api::urls,
-    graphql::front_page_query::*,
-    state::{Context, Model, Msg},
-    view::{
-        self, human_age, legacy, pretty_authors, set_title, tags_chiclet, view_header,
-        view_search_pager,
-    },
-};
-
-pub(super) fn view(model: &Model) -> Node<Msg> {
-    let content = match &model.context {
-        Context::None => div![h1!["Loading"]],
-        Context::Thread(thread_set) => legacy::thread(thread_set),
-        Context::ThreadResult(thread) => view::thread(thread),
-        Context::Search(search_results) => {
-            legacy::mobile_search_results(&model.query, search_results)
-        }
-        Context::SearchResult {
-            query,
-            results,
-            count,
-            pager,
-        } => search_results(&query, results.as_slice(), *count, pager),
-    };
-    div![
-        view_header(&model.query, &model.refreshing_state),
-        content,
-        view_header(&model.query, &model.refreshing_state),
-    ]
-}
-
-fn search_results(
-    query: &str,
-    results: &[FrontPageQuerySearchNodes],
-    count: usize,
-    pager: &FrontPageQuerySearchPageInfo,
-) -> Node<Msg> {
-    if query.is_empty() {
-        set_title("all mail");
-    } else {
-        set_title(query);
-    }
-    let rows = results.iter().map(|r| {
-        let tid = r.thread.clone();
-        let datetime = human_age(r.timestamp as i64);
-        a![
-            C!["has-text-light"],
-            attrs! {
-                At::Href => urls::thread(&tid)
-            },
-            div![
-                C!["row"],
-                div![C!["subject"], &r.subject],
-                span![C!["from", "is-size-7"], pretty_authors(&r.authors)],
-                div![
-                    span![C!["is-size-7"], tags_chiclet(&r.tags, true)],
-                    span![C!["is-size-7", "float-right", "date"], datetime]
-                ]
-            ]
-        ]
-    });
-    div![
-        C!["search-results"],
-        view_search_pager(count, pager),
-        rows,
-        view_search_pager(count, pager),
-    ]
-}

web/src/view/tablet.rs

@@ -1,30 +0,0 @@
-use seed::{prelude::*, *};
-
-use crate::{
-    state::{Context, Model, Msg},
-    view::{self, view_header, view_search_results},
-};
-
-pub(super) fn view(model: &Model) -> Node<Msg> {
-    // Do two queries, one without `unread` so it loads fast, then a second with unread.
-    let content = match &model.context {
-        Context::None => div![h1!["Loading"]],
-        Context::Thread(_) => unimplemented!("tablet legacy thread view"),
-        Context::ThreadResult(thread) => view::thread(thread),
-        Context::Search(_) => unimplemented!("tablet legacy search results view"),
-        Context::SearchResult {
-            query,
-            results,
-            count,
-            pager,
-        } => view_search_results(&query, results.as_slice(), *count, pager),
-    };
-    div![
-        C!["main-content"],
-        div![
-            view_header(&model.query, &model.refreshing_state),
-            content,
-            view_header(&model.query, &model.refreshing_state),
-        ]
-    ]
-}