chore: Release

Show original/delivered To if no xinu.tv addresses in To/CC fields

.gitea/workflows/rust.yml

@@ -0,0 +1,67 @@
+on: [push]
+
+name: Continuous integration
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo test
+
+  trunk:
+    name: Trunk
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+            toolchain: stable
+            target: wasm32-unknown-unknown
+      - run: cargo install trunk
+      - run: cd web; trunk build
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+      - name: Rustfmt Check
+        uses: actions-rust-lang/rustfmt@v1
+
+  build:
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build
+
+  udeps:
+    name: Disallow unused dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+
+      - name: Run cargo-udeps
+        uses: aig787/cargo-udeps-action@v1
+        with:
+          version: 'latest'
+          args: '--all-targets'
+

Cargo.toml

@@ -3,6 +3,14 @@ resolver = "2"
 default-members = ["server"]
 members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
 
+[workspace.package]
+authors = ["Bill Thiede <git@xinu.tv>"]
+edition = "2021"
+license = "UNLICENSED"
+publish = ["xinu"]
+version = "0.10.0"
+repository = "https://git.z.xinu.tv/wathiede/letterbox"
+
 [profile.dev]
 opt-level = 1
 

Justfile

@@ -0,0 +1,19 @@
+export CARGO_INCREMENTAL := "0"
+export RUSTFLAGS := "-D warnings"
+
+default:
+    @echo "Run: just patch|minor|major"
+
+major: (_release "major")
+minor: (_release "minor")
+patch: (_release "patch")
+
+sqlx-prepare:
+	cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true
+
+pull:
+    git pull
+
+
+_release level: pull sqlx-prepare
+    cargo-release release  -x {{ level }} --workspace --no-confirm --registry=xinu

notmuch/Cargo.toml

@@ -1,18 +1,24 @@
 [package]
-name = "notmuch"
+name = "letterbox-notmuch"
-version = "0.0.115"
+exclude = ["/testdata"]
-edition = "2021"
+description = "Wrapper for calling notmuch cli"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 log = "0.4.14"
+mailparse = "0.16.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
-thiserror = "1.0.30"
+thiserror = "2.0.0"
 tracing = "0.1.41"
 
 [dev-dependencies]
-itertools = "0.10.1"
+itertools = "0.14.0"
 pretty_assertions = "1"
 rayon = "1.5"

notmuch/src/lib.rs

@@ -207,6 +207,7 @@
 //! ```
 
 use std::{
+    collections::HashMap,
     ffi::OsStr,
     io::{self},
     path::{Path, PathBuf},
@@ -270,6 +271,12 @@ pub struct Headers {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub bcc: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "Delivered-To")]
+    pub delivered_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "X-Original-To")]
+    pub x_original_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub reply_to: Option<String>,
     pub date: String,
 }
@@ -459,6 +466,8 @@ pub enum NotmuchError {
     StringUtf8Error(#[from] std::string::FromUtf8Error),
     #[error("failed to parse str as int")]
     ParseIntError(#[from] std::num::ParseIntError),
+    #[error("failed to parse mail: {0}")]
+    MailParseError(#[from] mailparse::MailParseError),
 }
 
 #[derive(Default)]
@@ -605,6 +614,80 @@ impl Notmuch {
         Ok(serde_json::from_slice(&res)?)
     }
 
+    #[instrument(skip_all)]
+    pub fn unread_recipients(&self) -> Result<HashMap<String, usize>, NotmuchError> {
+        let slice = self.run_notmuch([
+            "show",
+            "--include-html=false",
+            "--entire-thread=false",
+            "--body=false",
+            "--format=json",
+            // Arbitrary limit to prevent too much work
+            "--limit=1000",
+            "is:unread",
+        ])?;
+        // Notmuch returns JSON with invalid unicode.  So we lossy convert it to a string here and
+        // use that for parsing in rust.
+        let s = String::from_utf8_lossy(&slice);
+        let mut deserializer = serde_json::Deserializer::from_str(&s);
+        deserializer.disable_recursion_limit();
+        let ts: ThreadSet = serde::de::Deserialize::deserialize(&mut deserializer)?;
+        deserializer.end()?;
+        let mut r = HashMap::new();
+        fn collect_from_thread_node(
+            r: &mut HashMap<String, usize>,
+            tn: &ThreadNode,
+        ) -> Result<(), NotmuchError> {
+            let Some(msg) = &tn.0 else {
+                return Ok(());
+            };
+            let mut addrs = vec![];
+            let hdr = &msg.headers.to;
+            if let Some(to) = hdr {
+                addrs.push(to);
+            } else {
+                let hdr = &msg.headers.x_original_to;
+                if let Some(to) = hdr {
+                    addrs.push(to);
+                } else {
+                    let hdr = &msg.headers.delivered_to;
+                    if let Some(to) = hdr {
+                        addrs.push(to);
+                    };
+                };
+            };
+            let hdr = &msg.headers.cc;
+            if let Some(cc) = hdr {
+                addrs.push(cc);
+            };
+            for recipient in addrs {
+                mailparse::addrparse(&recipient)?
+                    .into_inner()
+                    .iter()
+                    .for_each(|a| {
+                        let mailparse::MailAddr::Single(si) = a else {
+                            return;
+                        };
+                        let addr = &si.addr;
+
+                        if addr == "couchmoney@gmail.com" || addr.ends_with("@xinu.tv") {
+                            *r.entry(addr.to_lowercase()).or_default() += 1;
+                        }
+                    });
+            }
+            Ok(())
+        }
+        for t in ts.0 {
+            for tn in t.0 {
+                collect_from_thread_node(&mut r, &tn)?;
+                for sub_tn in tn.1 {
+                    collect_from_thread_node(&mut r, &sub_tn)?;
+                }
+            }
+        }
+        Ok(r)
+    }
+
     fn run_notmuch<I, S>(&self, args: I) -> Result<Vec<u8>, NotmuchError>
     where
         I: IntoIterator<Item = S>,

notmuch/tests/allmail.rs

@@ -4,7 +4,7 @@ use std::{
     time::Instant,
 };
 
-use notmuch::Notmuch;
+use letterbox_notmuch::Notmuch;
 use rayon::iter::{ParallelBridge, ParallelIterator};
 
 #[test]

procmail2notmuch/Cargo.toml

@@ -1,7 +1,12 @@
 [package]
-name = "procmail2notmuch"
+name = "letterbox-procmail2notmuch"
-version = "0.0.115"
+description = "Tool for generating notmuch rules from procmail"
-edition = "2021"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

server/.sqlx/query-383221a94bc3746322ba78e41cde37994440ee67dc32e88d2394c51211bde6cd.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n    date,\n    is_read,\n    link,\n    site,\n    summary,\n    title,\n    name,\n    homepage\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    uid = $1\n",
+  "query": "SELECT\n    date,\n    is_read,\n    link,\n    site,\n    summary,\n    clean_summary,\n    title,\n    name,\n    homepage\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug\nWHERE\n    uid = $1\n",
   "describe": {
     "columns": [
       {
@@ -30,16 +30,21 @@
       },
       {
         "ordinal": 5,
-        "name": "title",
+        "name": "clean_summary",
         "type_info": "Text"
       },
       {
         "ordinal": 6,
-        "name": "name",
+        "name": "title",
         "type_info": "Text"
       },
       {
         "ordinal": 7,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 8,
         "name": "homepage",
         "type_info": "Text"
       }
@@ -57,8 +62,9 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "113694cd5bf0d2582ff3a635776daa608fe88abe1185958c4215646c92335afb"
+  "hash": "383221a94bc3746322ba78e41cde37994440ee67dc32e88d2394c51211bde6cd"
 }

server/.sqlx/query-3d271b404f06497a5dcde68cf6bf07291d70fa56058ea736ac24e91d33050c04.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE\n    search_summary IS NULL\n    -- TODO remove AND link ~ '^<'\nORDER BY\n    ROW_NUMBER() OVER (PARTITION BY site ORDER BY date DESC)\nLIMIT 100;\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "link",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "clean_summary",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "3d271b404f06497a5dcde68cf6bf07291d70fa56058ea736ac24e91d33050c04"
+}

server/.sqlx/query-8c1b3c78649135e98b89092237750088433f7ff1b7c2ddeedec553406ea9f203.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) AS count\nFROM\n    post\nWHERE\n    (\n        $1::text IS NULL\n        OR site = $1\n    )\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n        $3::text IS NULL\n        OR TO_TSVECTOR('english', search_summary)\n        @@ WEBSEARCH_TO_TSQUERY('english', $3)\n    )\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8c1b3c78649135e98b89092237750088433f7ff1b7c2ddeedec553406ea9f203"
+}

server/.sqlx/query-e118f546c628661023aa25803bb29affb6cd25eca63246e5ace5b90a845d76ac.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n    COUNT(*) count\nFROM\n    post\nWHERE\n    (\n        $1 :: text IS NULL\n        OR site = $1\n    )\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $3 :: text IS NULL\n      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $3)\n    )\n",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Bool",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "e118f546c628661023aa25803bb29affb6cd25eca63246e5ace5b90a845d76ac"
-}

server/.sqlx/query-ef8327f039dbfa8f4e59b7a77a6411252a346bf51cf940024a17d9fbb2df173c.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE post SET search_summary = $1 WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ef8327f039dbfa8f4e59b7a77a6411252a346bf51cf940024a17d9fbb2df173c"
+}

server/.sqlx/query-fc4607f02cc76a5f3a6629cce4507c74f52ae44820897b47365da3f339d1da06.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $5 :: text IS NULL\n      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $5)\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
+  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n    AND (\n      $5 :: text IS NULL\n      OR to_tsvector('english', search_summary) @@ websearch_to_tsquery('english', $5)\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
   "describe": {
     "columns": [
       {
@@ -52,5 +52,5 @@
       true
     ]
   },
-  "hash": "99114d4840067acb12d9a41ef036bdd8ecf87cfdde8ce4985821485816af5213"
+  "hash": "fc4607f02cc76a5f3a6629cce4507c74f52ae44820897b47365da3f339d1da06"
 }

server/Cargo.toml

@@ -1,51 +1,58 @@
 [package]
 name = "letterbox-server"
-version = "0.0.115"
-edition = "2021"
 default-run = "letterbox-server"
+description = "Backend for letterbox"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ammonia = "3.3.0"
+ammonia = "4.0.0"
 anyhow = "1.0.79"
 async-graphql = { version = "7", features = ["log"] }
 async-graphql-rocket = "7"
 async-trait = "0.1.81"
-build-info = "0.0.38"
+build-info = "0.0.40"
 cacher = { version = "0.1.0", registry = "xinu" }
 chrono = "0.4.39"
 clap = { version = "4.5.23", features = ["derive"] }
-css-inline = "0.13.0"
+css-inline = "0.14.0"
+futures = "0.3.31"
 html-escape = "0.2.13"
 linkify = "0.10.0"
 log = "0.4.17"
-lol_html = "1.2.0"
+lol_html = "2.0.0"
-mailparse = "0.15.0"
+mailparse = "0.16.0"
 maplit = "1.0.2"
 memmap = "0.7.0"
-notmuch = { path = "../notmuch" }
+opentelemetry = "0.28.0"
-opentelemetry = "0.27.1"
+regex = "1.11.1"
 reqwest = { version = "0.12.7", features = ["blocking"] }
 rocket = { version = "0.5.0-rc.2", features = ["json"] }
 rocket_cors = "0.6.0"
-scraper = "0.20.0"
+scraper = "0.23.0"
 serde = { version = "1.0.147", features = ["derive"] }
 serde_json = "1.0.87"
-shared = { path = "../shared" }
 sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
 tantivy = { version = "0.22.0", optional = true }
-thiserror = "1.0.37"
+thiserror = "2.0.0"
 tokio = "1.26.0"
 tracing = "0.1.41"
 url = "2.5.2"
 urlencoding = "2.1.3"
 #xtracing = { path = "../../xtracing" }
 #xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
-xtracing = { version = "0.1.0", registry = "xinu" }
+xtracing = { version = "0.3.0", registry = "xinu" }
+letterbox-notmuch = { version = "0.10.0", path = "../notmuch", registry = "xinu" }
+letterbox-shared = { version = "0.10.0", path = "../shared", registry = "xinu" }
 
 [build-dependencies]
-build-info-build = "0.0.38"
+build-info-build = "0.0.40"
 
 [features]
 #default = [ "tantivy" ]

server/Rocket.toml

@@ -11,4 +11,4 @@ port = 9345
 #log_level = "critical"
 newsreader_database_url = "postgres://newsreader@nixos-07.h.xinu.tv/newsreader"
 newsreader_tantivy_db_path = "../target/database/newsreader"
-slurp_cache_path = "/net/nasx/x/letterbox/slurp"
+slurp_cache_path = "/tmp/letterbox/slurp"

server/migrations/20250127192339_nzb_posts_created_at_idx.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP INDEX nzb_posts_created_at_idx;

server/migrations/20250127192339_nzb_posts_created_at_idx.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX nzb_posts_created_at_idx ON nzb_posts USING btree (created_at);

server/migrations/20250128234348_add-search-summary.down.sql

@@ -0,0 +1,15 @@
+-- Add down migration script here
+BEGIN;
+DROP INDEX IF EXISTS post_search_summary_idx;
+ALTER TABLE post DROP search_summary;
+
+-- CREATE INDEX post_summary_idx ON post USING gin (to_tsvector(
+--     'english',
+--     regexp_replace(
+--         regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+--         '\s+',
+--         ' ',
+--         'g'
+--     )
+-- ));
+COMMIT;

server/migrations/20250128234348_add-search-summary.up.sql

@@ -0,0 +1,14 @@
+-- Add up migration script here
+BEGIN;
+DROP INDEX IF EXISTS post_summary_idx;
+ALTER TABLE post ADD search_summary TEXT;
+CREATE INDEX post_search_summary_idx ON post USING gin (
+    to_tsvector('english', search_summary)
+);
+UPDATE post SET search_summary = regexp_replace(
+    regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+    '\s+',
+    ' ',
+    'g'
+);
+COMMIT;

server/sql/count.sql

@@ -1,10 +1,9 @@
-SELECT
+SELECT COUNT(*) AS count
-    COUNT(*) count
 FROM
     post
 WHERE
     (
-        $1 :: text IS NULL
+        $1::text IS NULL
         OR site = $1
     )
     AND (
@@ -12,6 +11,7 @@ WHERE
         OR NOT is_read
     )
     AND (
-      $3 :: text IS NULL
+        $3::text IS NULL
-      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $3)
+        OR TO_TSVECTOR('english', search_summary)
+        @@ WEBSEARCH_TO_TSQUERY('english', $3)
     )

server/sql/need-search-summary.sql

@@ -0,0 +1,13 @@
+SELECT
+    p.id,
+    link,
+    clean_summary
+FROM
+    post AS p
+INNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
+WHERE
+    search_summary IS NULL
+    -- TODO remove AND link ~ '^<'
+ORDER BY
+    ROW_NUMBER() OVER (PARTITION BY site ORDER BY date DESC)
+LIMIT 100;

server/sql/thread.sql

@@ -4,11 +4,12 @@ SELECT
     link,
     site,
     summary,
+    clean_summary,
     title,
     name,
     homepage
 FROM
-    post p
+    post AS p
-    JOIN feed f ON p.site = f.slug
+INNER JOIN feed AS f ON p.site = f.slug
 WHERE
     uid = $1

server/sql/threads.sql

@@ -16,7 +16,7 @@ WHERE
     )
     AND (
       $5 :: text IS NULL
-      OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $5)
+      OR to_tsvector('english', search_summary) @@ websearch_to_tsquery('english', $5)
     )
 ORDER BY
     date DESC,

server/sql/too-long-for-search.sql

@@ -0,0 +1,13 @@
+select t.id, tt.tokid, tt.alias, length(t.token), t.token from (
+    select  id, (ts_parse('default', 
+    -- regexp_replace(
+    --     regexp_replace(summary, '<[^>]+>', ' ', 'g'),
+    --     '\s+',
+    --     ' ',
+    --     'g'
+    -- )
+        summary
+    )).* from post) t
+    inner join ts_token_type('default') tt
+       on t.tokid = tt.tokid
+    where length(token) >= 2*1024;

server/src/bin/inline_html.rs

@@ -0,0 +1,21 @@
+use std::fs;
+
+use url::Url;
+
+fn main() -> anyhow::Result<()> {
+    println!("PWD: {}", std::env::current_dir()?.display());
+    let _url = "https://slashdot.org/story/25/01/24/1813201/walgreens-replaced-fridge-doors-with-smart-screens-its-now-a-200-million-fiasco?utm_source=rss1.0mainlinkanon&utm_medium=feed";
+    let _url = "https://hackaday.com/2025/01/24/hackaday-podcast-episode-305-caustic-clocks-practice-bones-and-brick-layers/";
+    let _url =    "https://theonion.com/monster-devastated-to-see-film-depicting-things-he-told-guillermo-del-toro-in-confidence/";
+    let _url = "https://trofi.github.io/posts/330-another-nix-language-nondeterminism-example.html";
+    let _url = "https://blog.cloudflare.com/ddos-threat-report-for-2024-q4/";
+    let url = "https://trofi.github.io/posts/330-another-nix-language-nondeterminism-example.html";
+    let body = reqwest::blocking::get(url)?.text()?;
+    let output = "/tmp/h2md/output.html";
+    let inliner = css_inline::CSSInliner::options()
+        .base_url(Url::parse(url).ok())
+        .build();
+    let inlined = inliner.inline(&body)?;
+    fs::write(output, inlined)?;
+    Ok(())
+}

server/src/bin/letterbox-server.rs

@@ -7,6 +7,8 @@ use std::{error::Error, io::Cursor, str::FromStr};
 
 use async_graphql::{extensions, http::GraphiQLSource, EmptySubscription, Schema};
 use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
+use cacher::FilesystemCacher;
+use letterbox_notmuch::{Notmuch, NotmuchError, ThreadSet};
 #[cfg(feature = "tantivy")]
 use letterbox_server::tantivy::TantivyConnection;
 use letterbox_server::{
@@ -15,7 +17,6 @@ use letterbox_server::{
     graphql::{Attachment, GraphqlSchema, Mutation, QueryRoot},
     nm::{attachment_bytes, cid_attachment_bytes},
 };
-use notmuch::{Notmuch, NotmuchError, ThreadSet};
 use rocket::{
     fairing::AdHoc,
     http::{ContentType, Header},
@@ -178,7 +179,7 @@ async fn graphql_request(
 async fn main() -> Result<(), Box<dyn Error>> {
     let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
     build_info::build_info!(fn bi);
-    info!("Build Info: {}", shared::build_version(bi));
+    info!("Build Info: {}", letterbox_shared::build_version(bi));
     let allowed_origins = AllowedOrigins::all();
     let cors = rocket_cors::CorsOptions {
         allowed_origins,
@@ -194,7 +195,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     let rkt = rocket::build()
         .mount(
-            shared::urls::MOUNT_POINT,
+            letterbox_shared::urls::MOUNT_POINT,
             routes![
                 original,
                 show_pretty,
@@ -220,9 +221,10 @@ async fn main() -> Result<(), Box<dyn Error>> {
     #[cfg(feature = "tantivy")]
     let tantivy_conn = TantivyConnection::new(&config.newsreader_tantivy_db_path)?;
 
+    let cacher = FilesystemCacher::new(&config.slurp_cache_path)?;
     let schema = Schema::build(QueryRoot, Mutation, EmptySubscription)
         .data(Notmuch::default())
-        .data(config)
+        .data(cacher)
         .data(pool.clone());
 
     #[cfg(feature = "tantivy")]

server/src/custom.css

@@ -1,8 +0,0 @@
-pre {
-  background-color: var(--color-bg);
-  color: var(--color-text);
-}
-
-code {
-  background-color: var(--color-bg-secondary);
-}

server/src/error.rs

@@ -10,7 +10,7 @@ use crate::TransformError;
 #[derive(Error, Debug)]
 pub enum ServerError {
     #[error("notmuch: {0}")]
-    NotmuchError(#[from] notmuch::NotmuchError),
+    NotmuchError(#[from] letterbox_notmuch::NotmuchError),
     #[error("flatten")]
     FlattenError,
     #[error("mail parse error: {0}")]

server/src/graphql.rs

@@ -5,8 +5,9 @@ use async_graphql::{
     Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
     SimpleObject, Union,
 };
+use cacher::FilesystemCacher;
+use letterbox_notmuch::Notmuch;
 use log::info;
-use notmuch::Notmuch;
 use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPool;
 use tokio::join;
@@ -14,7 +15,7 @@ use tracing::instrument;
 
 #[cfg(feature = "tantivy")]
 use crate::tantivy::TantivyConnection;
-use crate::{config::Config, newsreader, nm, Query};
+use crate::{newsreader, nm, Query};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -94,6 +95,10 @@ pub struct Message {
     pub to: Vec<Email>,
     // All CC headers found in email
     pub cc: Vec<Email>,
+    // X-Original-To header found in email
+    pub x_original_to: Option<Email>,
+    // Delivered-To header found in email
+    pub delivered_to: Option<Email>,
     // First Subject header found in email
     pub subject: Option<String>,
     // Parsed Date header, if found and valid
@@ -282,10 +287,10 @@ pub struct QueryRoot;
 impl QueryRoot {
     async fn version<'ctx>(&self, _ctx: &Context<'ctx>) -> Result<String, Error> {
         build_info::build_info!(fn bi);
-        Ok(shared::build_version(bi))
+        Ok(letterbox_shared::build_version(bi))
     }
     #[instrument(skip_all, fields(query=query))]
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn count<'ctx>(&self, ctx: &Context<'ctx>, query: String) -> Result<usize, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -305,10 +310,49 @@ impl QueryRoot {
         info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count} tantivy count {tantivy_count} total {total}");
         Ok(total)
     }
+    async fn catchup<'ctx>(
+        &self,
+        ctx: &Context<'ctx>,
+        query: String,
+    ) -> Result<Vec<String>, Error> {
+        let nm = ctx.data_unchecked::<Notmuch>();
+        let pool = ctx.data_unchecked::<PgPool>();
+        let query: Query = query.parse()?;
+        // TODO: implement optimized versions of fetching just IDs
+        let newsreader_fut = newsreader_search(pool, None, None, None, None, &query);
+        let notmuch_fut = notmuch_search(nm, None, None, None, None, &query);
+        let (newsreader_results, notmuch_results) = join!(newsreader_fut, notmuch_fut);
+
+        let newsreader_results = newsreader_results?;
+        let notmuch_results = notmuch_results?;
+        info!(
+            "newsreader_results ({}) notmuch_results ({})",
+            newsreader_results.len(),
+            notmuch_results.len(),
+        );
+
+        let mut results: Vec<_> = newsreader_results
+            .into_iter()
+            .chain(notmuch_results)
+            .collect();
+        // The leading '-' is to reverse sort
+        results.sort_by_key(|item| match item {
+            ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
+            ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
+        });
+        let ids = results
+            .into_iter()
+            .map(|r| match r {
+                ThreadSummaryCursor::Newsreader(_, ts) => ts.thread,
+                ThreadSummaryCursor::Notmuch(_, ts) => ts.thread,
+            })
+            .collect();
+        Ok(ids)
+    }
 
     // TODO: this function doesn't get parallelism, possibly because notmuch is sync and blocks,
     // rewrite that with tokio::process:Command
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn search<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -466,7 +510,7 @@ impl QueryRoot {
         .await?)
     }
 
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -475,11 +519,11 @@ impl QueryRoot {
         tags.append(&mut nm::tags(nm, needs_unread)?);
         Ok(tags)
     }
-    #[instrument(skip_all, fields(thread_id=thread_id, request_id=request_id()))]
+    #[instrument(skip_all, fields(thread_id=thread_id, rid=request_id()))]
     async fn thread<'ctx>(&self, ctx: &Context<'ctx>, thread_id: String) -> Result<Thread, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
+        let cacher = ctx.data_unchecked::<FilesystemCacher>();
         let pool = ctx.data_unchecked::<PgPool>();
-        let config = ctx.data_unchecked::<Config>();
         let debug_content_tree = ctx
             .look_ahead()
             .field("messages")
@@ -487,7 +531,7 @@ impl QueryRoot {
             .field("contentTree")
             .exists();
         if newsreader::is_newsreader_thread(&thread_id) {
-            Ok(newsreader::thread(config, pool, thread_id).await?)
+            Ok(newsreader::thread(cacher, pool, thread_id).await?)
         } else {
             Ok(nm::thread(nm, pool, thread_id, debug_content_tree).await?)
         }
@@ -552,7 +596,7 @@ async fn tantivy_search(
 pub struct Mutation;
 #[Object]
 impl Mutation {
-    #[instrument(skip_all, fields(query=query, unread=unread, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, unread=unread, rid=request_id()))]
     async fn set_read_status<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -571,7 +615,7 @@ impl Mutation {
         nm::set_read_status(nm, &query, unread).await?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_add<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -583,7 +627,7 @@ impl Mutation {
         nm.tag_add(&tag, &query)?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_remove<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -606,14 +650,16 @@ impl Mutation {
 
         Ok(true)
     }
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn refresh<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
+        let cacher = ctx.data_unchecked::<FilesystemCacher>();
+        let pool = ctx.data_unchecked::<PgPool>();
         info!("{}", String::from_utf8_lossy(&nm.new()?));
+        newsreader::refresh(pool, cacher).await?;
         #[cfg(feature = "tantivy")]
         {
             let tantivy = ctx.data_unchecked::<TantivyConnection>();
-            let pool = ctx.data_unchecked::<PgPool>();
             // TODO: parallelize
             tantivy.refresh(pool).await?;
         }

server/src/lib.rs

@@ -7,22 +7,29 @@ pub mod nm;
 #[cfg(feature = "tantivy")]
 pub mod tantivy;
 
-use std::{collections::HashMap, convert::Infallible, fmt, str::FromStr, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    convert::Infallible,
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
 
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
 use css_inline::{CSSInliner, InlineError, InlineOptions};
 use linkify::{LinkFinder, LinkKind};
-use log::{error, info, warn};
+use log::{debug, error, info, warn};
 use lol_html::{
     element, errors::RewritingError, html_content::ContentType, rewrite_str, text,
     RewriteStrSettings,
 };
 use maplit::{hashmap, hashset};
+use regex::Regex;
+use reqwest::StatusCode;
 use scraper::{Html, Selector};
 use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
-use tokio::sync::Mutex;
 use url::Url;
 
 use crate::{
@@ -58,6 +65,8 @@ pub enum TransformError {
     ReqwestError(#[from] reqwest::Error),
     #[error("failed to parse HTML: {0}")]
     HtmlParsingError(String),
+    #[error("got a retryable error code {0} for {1}")]
+    RetryableHttpStatusError(StatusCode, String),
 }
 
 struct SanitizeHtml<'a> {
@@ -88,70 +97,49 @@ struct StripHtml;
 
 #[async_trait]
 impl Transformer for StripHtml {
-    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("StripHtml should_run {link:?} {}", html.contains("<"));
         // Lame test
         html.contains("<")
     }
-    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("StripHtml {link:?}");
         let mut text = String::new();
-        let element_content_handlers = vec![text!("*", |t| {
+        let element_content_handlers = vec![
-            text += t.as_str();
+            element!("style", |el| {
-            Ok(())
+                el.remove();
-        })];
+                Ok(())
-        let _ = rewrite_str(
+            }),
+            element!("script", |el| {
+                el.remove();
+                Ok(())
+            }),
+        ];
+        let html = rewrite_str(
             html,
             RewriteStrSettings {
                 element_content_handlers,
                 ..RewriteStrSettings::default()
             },
         )?;
+        let element_content_handlers = vec![text!("*", |t| {
+            text += t.as_str();
+            Ok(())
+        })];
+        let _ = rewrite_str(
+            &html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
+        let re = Regex::new(r"\s+").expect("failed to parse regex");
+        let text = re.replace_all(&text, " ").to_string();
 
         Ok(text)
     }
 }
 
-struct InlineRemoteStyle<'a> {
-    base_url: &'a Option<Url>,
-}
-
-#[async_trait]
-impl<'a> Transformer for InlineRemoteStyle<'a> {
-    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
-        let css = concat!(
-            "/* chrome-default.css */\n",
-            include_str!("chrome-default.css"),
-            "\n/* mvp.css */\n",
-            include_str!("mvp.css"),
-            "\n/* Xinu Specific overrides */\n",
-            include_str!("custom.css"),
-        );
-        let inline_opts = InlineOptions {
-            //inline_style_tags: true,
-            //keep_style_tags: false,
-            //keep_link_tags: true,
-            base_url: self.base_url.clone(),
-            //load_remote_stylesheets: true,
-            //preallocate_node_capacity: 32,
-            ..InlineOptions::default()
-        };
-
-        //info!("HTML:\n{html}");
-        info!("base_url: {:#?}", self.base_url);
-        Ok(
-            match CSSInliner::options()
-                .base_url(self.base_url.clone())
-                .build()
-                .inline(&html)
-            {
-                Ok(inlined_html) => inlined_html,
-                Err(err) => {
-                    error!("failed to inline remote CSS: {err}");
-                    html.to_string()
-                }
-            },
-        )
-    }
-}
 struct InlineStyle;
 
 #[async_trait]
@@ -160,10 +148,10 @@ impl Transformer for InlineStyle {
         let css = concat!(
             "/* chrome-default.css */\n",
             include_str!("chrome-default.css"),
-            "\n/* mvp.css */\n",
+            //"\n/* mvp.css */\n",
-            include_str!("mvp.css"),
+            //include_str!("mvp.css"),
-            "\n/* Xinu Specific overrides */\n",
+            //"\n/* Xinu Specific overrides */\n",
-            include_str!("custom.css"),
+            //include_str!("custom.css"),
         );
         let inline_opts = InlineOptions {
             inline_style_tags: true,
@@ -269,13 +257,13 @@ impl Transformer for AddOutlink {
     }
 }
 
-struct SlurpContents {
+struct SlurpContents<'c> {
-    cacher: Arc<Mutex<FilesystemCacher>>,
+    cacher: &'c FilesystemCacher,
     inline_css: bool,
     site_selectors: HashMap<String, Vec<Selector>>,
 }
 
-impl SlurpContents {
+impl<'c> SlurpContents<'c> {
     fn get_selectors(&self, link: &Url) -> Option<&[Selector]> {
         for (host, selector) in self.site_selectors.iter() {
             if link.host_str().map(|h| h.contains(host)).unwrap_or(false) {
@@ -287,36 +275,79 @@ impl SlurpContents {
 }
 
 #[async_trait]
-impl Transformer for SlurpContents {
+impl<'c> Transformer for SlurpContents<'c> {
-    fn should_run(&self, link: &Option<Url>, _: &str) -> bool {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("SlurpContents should_run {link:?}");
+        let mut will_slurp = false;
         if let Some(link) = link {
-            return self.get_selectors(link).is_some();
+            will_slurp = self.get_selectors(link).is_some();
         }
-        false
+        if !will_slurp && self.inline_css {
+            return InlineStyle {}.should_run(link, html);
+        }
+        will_slurp
     }
     async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("SlurpContents {link:?}");
+        let retryable_status: HashSet<StatusCode> = vec![
+            StatusCode::UNAUTHORIZED,
+            StatusCode::FORBIDDEN,
+            StatusCode::REQUEST_TIMEOUT,
+            StatusCode::TOO_MANY_REQUESTS,
+        ]
+        .into_iter()
+        .collect();
+        if let Some(test_link) = link {
+            // If SlurpContents is configured for inline CSS, but no
+            // configuration found for this site, use the local InlineStyle
+            // transform.
+            if self.inline_css && self.get_selectors(test_link).is_none() {
+                debug!("local inline CSS for {link:?}");
+                return InlineStyle {}.transform(link, html).await;
+            }
+        }
         let Some(link) = link else {
             return Ok(html.to_string());
         };
         let Some(selectors) = self.get_selectors(&link) else {
             return Ok(html.to_string());
         };
-        let cacher = self.cacher.lock().await;
+        let cacher = self.cacher;
         let body = if let Some(body) = cacher.get(link.as_str()) {
-            info!("cache hit for {link}");
             String::from_utf8_lossy(&body).to_string()
         } else {
-            let body = reqwest::get(link.as_str()).await?.text().await?;
+            let resp = reqwest::get(link.as_str()).await?;
+            let status = resp.status();
+            if status.is_server_error() || retryable_status.contains(&status) {
+                return Err(TransformError::RetryableHttpStatusError(
+                    status,
+                    link.to_string(),
+                ));
+            }
+            if !status.is_success() {
+                return Ok(html.to_string());
+            }
+            let body = resp.text().await?;
             cacher.set(link.as_str(), body.as_bytes());
             body
         };
         let body = Arc::new(body);
         let base_url = Some(link.clone());
         let body = if self.inline_css {
+            debug!("inlining CSS for {link}");
             let inner_body = Arc::clone(&body);
             let res = tokio::task::spawn_blocking(move || {
+                let css = concat!(
+                    "/* chrome-default.css */\n",
+                    include_str!("chrome-default.css"),
+                    "\n/* vars.css */\n",
+                    include_str!("../static/vars.css"),
+                    //"\n/* Xinu Specific overrides */\n",
+                    //include_str!("custom.css"),
+                );
                 let res = CSSInliner::options()
                     .base_url(base_url)
+                    .extra_css(Some(std::borrow::Cow::Borrowed(css)))
                     .build()
                     .inline(&inner_body);
 
@@ -337,6 +368,7 @@ impl Transformer for SlurpContents {
                 }
             }
         } else {
+            debug!("using body as-is for {link:?}");
             Arc::into_inner(body).expect("failed to take body out of Arc")
         };
 
@@ -650,7 +682,7 @@ fn compute_offset_limit(
     first: Option<i32>,
     last: Option<i32>,
 ) -> (i32, i32) {
-    let default_page_size = 100;
+    let default_page_size = 10000;
     match (after, before, first, last) {
         // Reasonable defaults
         (None, None, None, None) => (0, default_page_size),
@@ -741,7 +773,19 @@ impl Query {
         for uid in &self.uids {
             parts.push(uid.clone());
         }
-        parts.extend(self.remainder.clone());
+        for r in &self.remainder {
+            // Rewrite "to:" to include ExtraTo:. ExtraTo: is configured in
+            // notmuch-config to index Delivered-To and X-Original-To headers.
+            if r.starts_with("to:") {
+                parts.push("(".to_string());
+                parts.push(r.to_string());
+                parts.push("OR".to_string());
+                parts.push(r.replace("to:", "ExtraTo:"));
+                parts.push(")".to_string());
+            } else {
+                parts.push(r.to_string());
+            }
+        }
         parts.join(" ")
     }
 }
@@ -761,7 +805,17 @@ impl FromStr for Query {
             if word == "is:unread" {
                 unread_only = true
             } else if word.starts_with("tag:") {
-                tags.push(word["tag:".len()..].to_string());
+                let t = &word["tag:".len()..];
+                // Per-address emails are faked as `tag:@<domain>/<username>`, rewrite to `to:` form
+                if t.starts_with('@') && t.contains('.') {
+                    let t = match t.split_once('/') {
+                        None => format!("to:{t}"),
+                        Some((domain, user)) => format!("to:{user}{domain}"),
+                    };
+                    remainder.push(t);
+                } else {
+                    tags.push(t.to_string());
+                };
 
                 /*
                             } else if word.starts_with("tag:") {

server/src/newsreader.rs

@@ -1,23 +1,21 @@
-use std::sync::Arc;
+use std::collections::HashMap;
 
 use cacher::FilesystemCacher;
-use log::info;
+use futures::{stream::FuturesUnordered, StreamExt};
+use letterbox_shared::compute_color;
+use log::{error, info};
 use maplit::hashmap;
 use scraper::Selector;
-use shared::compute_color;
 use sqlx::postgres::PgPool;
-use tokio::sync::Mutex;
 use tracing::instrument;
 use url::Url;
 
 use crate::{
     clean_title, compute_offset_limit,
-    config::Config,
     error::ServerError,
     graphql::{Corpus, NewsPost, Tag, Thread, ThreadSummary},
-    thread_summary_from_row, AddOutlink, EscapeHtml, FrameImages, InlineRemoteStyle, Query,
+    thread_summary_from_row, AddOutlink, FrameImages, Query, SanitizeHtml, SlurpContents,
-    SanitizeHtml, SlurpContents, ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX,
+    StripHtml, ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX, NEWSREADER_THREAD_PREFIX,
-    NEWSREADER_THREAD_PREFIX,
 };
 
 pub fn is_newsreader_query(query: &Query) -> bool {
@@ -174,7 +172,7 @@ pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, Server
 
 #[instrument(name = "newsreader::thread", skip_all, fields(thread_id=%thread_id))]
 pub async fn thread(
-    config: &Config,
+    cacher: &FilesystemCacher,
     pool: &PgPool,
     thread_id: String,
 ) -> Result<Thread, ServerError> {
@@ -191,65 +189,12 @@ pub async fn thread(
     let site = r.name.unwrap_or("NO SITE".to_string());
     // TODO: remove the various places that have this as an Option
     let link = Some(Url::parse(&r.link)?);
-    let mut body = r.summary.unwrap_or("NO SUMMARY".to_string());
+    let mut body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
-    let cacher = Arc::new(Mutex::new(FilesystemCacher::new(&config.slurp_cache_path)?));
+    let body_transformers: Vec<Box<dyn Transformer>> = vec![
-    let body_tranformers: Vec<Box<dyn Transformer>> = vec![
         Box::new(SlurpContents {
             cacher,
-            // TODO: make this true when bulma is finally removed
+            inline_css: true,
-            inline_css: false,
+            site_selectors: slurp_contents_selectors(),
-            site_selectors: hashmap![
-                "atmeta.com".to_string() => vec![
-                    Selector::parse("div.entry-content").unwrap(),
-                ],
-                "blog.prusa3d.com".to_string() => vec![
-                    Selector::parse("article.content .post-block").unwrap(),
-                ],
-                "blog.cloudflare.com".to_string() => vec![
-                    Selector::parse(".author-lists .author-name-tooltip").unwrap(),
-                    Selector::parse(".post-full-content").unwrap()
-                ],
-                "blog.zsa.io".to_string() => vec![
-                    Selector::parse("section.blog-article").unwrap(),
-                ],
-                "engineering.fb.com".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "grafana.com".to_string() => vec![
-                    Selector::parse(".blog-content").unwrap(),
-                ],
-                "hackaday.com".to_string() => vec![
-                    Selector::parse("div.entry-featured-image").unwrap(),
-                    Selector::parse("div.entry-content").unwrap()
-                ],
-                "ingowald.blog".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "jvns.ca".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
-                "natwelch.com".to_string() => vec![
-                    Selector::parse("article div.prose").unwrap(),
-                ],
-                "rustacean-station.org".to_string() => vec![
-                    Selector::parse("article").unwrap(),
-                ],
-                "slashdot.org".to_string() => vec![
-                    Selector::parse("span.story-byline").unwrap(),
-                    Selector::parse("div.p").unwrap(),
-                ],
-                "trofi.github.io".to_string() => vec![
-                    Selector::parse("#content").unwrap(),
-                ],
-                "www.redox-os.org".to_string() => vec![
-                    Selector::parse("div.content").unwrap(),
-                ],
-                "www.smbc-comics.com".to_string() => vec![
-                    Selector::parse("img#cc-comic").unwrap(),
-                    Selector::parse("div#aftercomic img").unwrap(),
-                ],
-            ],
         }),
         Box::new(FrameImages),
         Box::new(AddOutlink),
@@ -260,7 +205,7 @@ pub async fn thread(
             base_url: &link,
         }),
     ];
-    for t in body_tranformers.iter() {
+    for t in body_transformers.iter() {
         if t.should_run(&link, &body) {
             body = t.transform(&link, &body).await?;
         }
@@ -308,3 +253,132 @@ pub async fn set_read_status<'ctx>(
     }
     Ok(true)
 }
+#[instrument(name = "newsreader::refresh", skip_all)]
+pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<bool, ServerError> {
+    async fn update_search_summary(
+        pool: &PgPool,
+        cacher: &FilesystemCacher,
+        link: String,
+        body: String,
+        id: i32,
+    ) -> Result<(), ServerError> {
+        let slurp_contents = SlurpContents {
+            cacher,
+            inline_css: true,
+            site_selectors: slurp_contents_selectors(),
+        };
+        let strip_html = StripHtml;
+
+        info!("adding {link} to search index");
+        let mut body = body;
+        if let Ok(link) = Url::parse(&link) {
+            let link = Some(link);
+            if slurp_contents.should_run(&link, &body) {
+                body = slurp_contents.transform(&link, &body).await?;
+            }
+        } else {
+            error!("failed to parse link: {}", link);
+        }
+        body = strip_html.transform(&None, &body).await?;
+        sqlx::query!(
+            "UPDATE post SET search_summary = $1 WHERE id = $2",
+            body,
+            id
+        )
+        .execute(pool)
+        .await?;
+        Ok(())
+    }
+
+    let mut unordered: FuturesUnordered<_> = sqlx::query_file!("sql/need-search-summary.sql",)
+        .fetch_all(pool)
+        .await?
+        .into_iter()
+        .filter_map(|r| {
+            let Some(body) = r.clean_summary else {
+                error!("clean_summary missing for {}", r.link);
+                return None;
+            };
+            let id = r.id;
+            Some(update_search_summary(pool, cacher, r.link, body, id))
+        })
+        .collect();
+
+    while let Some(res) = unordered.next().await {
+        //let res = res;
+        match res {
+            Ok(()) => {}
+            Err(err) => {
+                info!("failed refresh {err:?}");
+                // TODO:
+                //fd.error = Some(err);
+            }
+        };
+    }
+    Ok(true)
+}
+
+fn slurp_contents_selectors() -> HashMap<String, Vec<Selector>> {
+    hashmap![
+        "atmeta.com".to_string() => vec![
+            Selector::parse("div.entry-content").unwrap(),
+        ],
+        "blog.prusa3d.com".to_string() => vec![
+            Selector::parse("article.content .post-block").unwrap(),
+        ],
+        "blog.cloudflare.com".to_string() => vec![
+            Selector::parse(".author-lists .author-name-tooltip").unwrap(),
+            Selector::parse(".post-full-content").unwrap()
+        ],
+        "blog.zsa.io".to_string() => vec![
+            Selector::parse("section.blog-article").unwrap(),
+        ],
+        "engineering.fb.com".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "grafana.com".to_string() => vec![
+            Selector::parse(".blog-content").unwrap(),
+        ],
+        "hackaday.com".to_string() => vec![
+            Selector::parse("div.entry-featured-image").unwrap(),
+            Selector::parse("div.entry-content").unwrap()
+        ],
+        "ingowald.blog".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "jvns.ca".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
+        "natwelch.com".to_string() => vec![
+            Selector::parse("article div.prose").unwrap(),
+        ],
+        "rustacean-station.org".to_string() => vec![
+            Selector::parse("article").unwrap(),
+        ],
+        "slashdot.org".to_string() => vec![
+            Selector::parse("span.story-byline").unwrap(),
+            Selector::parse("div.p").unwrap(),
+        ],
+        "theonion.com".to_string() => vec![
+            // Single image joke w/ title
+            Selector::parse("article > section > div > figure").unwrap(),
+            // Single cartoon
+            Selector::parse("article > div > div > figure").unwrap(),
+            // Image at top of article
+            Selector::parse("article > header > div > div > figure").unwrap(),
+            // Article body
+            Selector::parse("article .entry-content > *").unwrap(),
+        ],
+        "trofi.github.io".to_string() => vec![
+            Selector::parse("#content").unwrap(),
+        ],
+        "www.redox-os.org".to_string() => vec![
+            Selector::parse("div.content").unwrap(),
+        ],
+        "www.smbc-comics.com".to_string() => vec![
+            Selector::parse("img#cc-comic").unwrap(),
+            Selector::parse("div#aftercomic img").unwrap(),
+        ],
+    ]
+}

server/src/nm.rs

@@ -1,14 +1,10 @@
-use std::{
+use std::{collections::HashMap, fs::File};
-    collections::HashMap,
-    fs::File,
-    hash::{DefaultHasher, Hash, Hasher},
-    time::Instant,
-};
 
+use letterbox_notmuch::Notmuch;
+use letterbox_shared::compute_color;
 use log::{error, info, warn};
 use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
 use memmap::MmapOptions;
-use notmuch::Notmuch;
 use sqlx::PgPool;
 use tracing::instrument;
 
@@ -43,7 +39,9 @@ pub fn is_notmuch_thread_or_id(id: &str) -> bool {
 }
 
 // TODO(wathiede): decide good error type
-pub fn threadset_to_messages(thread_set: notmuch::ThreadSet) -> Result<Vec<Message>, ServerError> {
+pub fn threadset_to_messages(
+    thread_set: letterbox_notmuch::ThreadSet,
+) -> Result<Vec<Message>, ServerError> {
     for t in thread_set.0 {
         for _tn in t.0 {}
     }
@@ -105,7 +103,6 @@ pub async fn search(
 
 #[instrument(name="nm::tags", skip_all, fields(needs_unread=needs_unread))]
 pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
-    let now = Instant::now();
     let unread_msg_cnt: HashMap<String, usize> = if needs_unread {
         // 10000 is an arbitrary number, if there's more than 10k unread messages, we'll
         //  get an inaccurate count.
@@ -121,13 +118,11 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     } else {
         HashMap::new()
     };
-    let tags = nm
+    let tags: Vec<_> = nm
         .tags()?
         .into_iter()
         .map(|tag| {
-            let mut hasher = DefaultHasher::new();
+            let hex = compute_color(&tag);
-            tag.hash(&mut hasher);
-            let hex = format!("#{:06x}", hasher.finish() % (1 << 24));
             let unread = if needs_unread {
                 *unread_msg_cnt.get(&tag).unwrap_or(&0)
             } else {
@@ -140,8 +135,24 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
                 unread,
             }
         })
+        .chain(
+            nm.unread_recipients()?
+                .into_iter()
+                .filter_map(|(name, unread)| {
+                    let Some(idx) = name.find('@') else {
+                        return None;
+                    };
+                    let name = format!("{}/{}", &name[idx..], &name[..idx]);
+                    let bg_color = compute_color(&name);
+                    Some(Tag {
+                        name,
+                        fg_color: "white".to_string(),
+                        bg_color,
+                        unread,
+                    })
+                }),
+        )
         .collect();
-    info!("Fetching tags took {} seconds", now.elapsed().as_secs_f32());
     Ok(tags)
 }
 
@@ -185,12 +196,14 @@ pub async fn thread(
 
         let to = email_addresses(&path, &m, "to")?;
         let cc = email_addresses(&path, &m, "cc")?;
+        let delivered_to = email_addresses(&path, &m, "delivered-to")?.pop();
+        let x_original_to = email_addresses(&path, &m, "x-original-to")?.pop();
         let subject = m.headers.get_first_value("subject");
         let timestamp = m
             .headers
             .get_first_value("date")
             .and_then(|d| mailparse::dateparse(&d).ok());
-        let cid_prefix = shared::urls::cid_prefix(None, &id);
+        let cid_prefix = letterbox_shared::urls::cid_prefix(None, &id);
         let base_url = None;
         let mut part_addr = Vec::new();
         part_addr.push(id.to_string());
@@ -223,7 +236,7 @@ pub async fn thread(
                         }
 
                         format!(
-                            r#"<p class="view-part-text-plain">{}</p>"#,
+                            r#"<p class="view-part-text-plain font-mono whitespace-pre-line">{}</p>"#,
                             // Trim newlines to prevent excessive white space at the beginning/end of
                             // presenation. Leave tabs and spaces incase plain text attempts to center a
                             // header on the first line.
@@ -304,6 +317,8 @@ pub async fn thread(
             body,
             path,
             attachments,
+            delivered_to,
+            x_original_to,
         });
     }
     messages.reverse();
@@ -578,7 +593,7 @@ fn flatten_body_parts(parts: &[Body]) -> Body {
         .map(|p| match p {
             Body::PlainText(PlainText { text, .. }) => {
                 format!(
-                    r#"<p class="view-part-text-plain">{}</p>"#,
+                    r#"<p class="view-part-text-plain font-mono whitespace-pre-line">{}</p>"#,
                     // Trim newlines to prevent excessive white space at the beginning/end of
                     // presenation. Leave tabs and spaces incase plain text attempts to center a
                     // header on the first line.
@@ -696,7 +711,6 @@ fn walk_attachments_inner<T, F: Fn(&ParsedMail, &[usize]) -> Option<T> + Copy>(
 fn extract_attachments(m: &ParsedMail, id: &str) -> Result<Vec<Attachment>, ServerError> {
     let mut attachments = Vec::new();
     for (idx, sp) in m.subparts.iter().enumerate() {
-        info!("sp: {:?}", sp.headers);
         if let Some(attachment) = extract_attachment(sp, id, &[idx]) {
             // Filter out inline attachements, they're flattened into the body of the message.
             if attachment.disposition == DispositionType::Attachment {

server/static/vars.css

@@ -0,0 +1,42 @@
+:root {
+    --active-brightness: 0.85;
+    --border-radius: 5px;
+    --box-shadow: 2px 2px 10px;
+    --color-accent: #118bee15;
+    --color-bg: #fff;
+    --color-bg-secondary: #e9e9e9;
+    --color-link: #118bee;
+    --color-secondary: #920de9;
+    --color-secondary-accent: #920de90b;
+    --color-shadow: #f4f4f4;
+    --color-table: #118bee;
+    --color-text: #000;
+    --color-text-secondary: #999;
+    --color-scrollbar: #cacae8;
+    --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+    --hover-brightness: 1.2;
+    --justify-important: center;
+    --justify-normal: left;
+    --line-height: 1.5;
+    /*
+  --width-card: 285px;
+  --width-card-medium: 460px;
+  --width-card-wide: 800px;
+*/
+    --width-content: 1080px;
+}
+
+@media (prefers-color-scheme: dark) {
+    :root[color-mode="user"] {
+        --color-accent: #0097fc4f;
+        --color-bg: #333;
+        --color-bg-secondary: #555;
+        --color-link: #0097fc;
+        --color-secondary: #e20de9;
+        --color-secondary-accent: #e20de94f;
+        --color-shadow: #bbbbbb20;
+        --color-table: #0097fc;
+        --color-text: #f7f7f7;
+        --color-text-secondary: #aaa;
+    }
+}

shared/Cargo.toml

@@ -1,11 +1,16 @@
 [package]
-name = "shared"
+name = "letterbox-shared"
-version = "0.0.115"
+description = "Shared module for letterbox"
-edition = "2021"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-build-info = "0.0.38"
+build-info = "0.0.40"
-notmuch = { path = "../notmuch" }
+letterbox-notmuch = { version = "0.10.0", path = "../notmuch", registry = "xinu" }
 serde = { version = "1.0.147", features = ["derive"] }

shared/src/lib.rs

@@ -1,7 +1,7 @@
 use std::hash::{DefaultHasher, Hash, Hasher};
 
 use build_info::{BuildInfo, VersionControl};
-use notmuch::SearchSummary;
+use letterbox_notmuch::SearchSummary;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug)]

web/Cargo.toml

@@ -1,16 +1,15 @@
 [package]
-version = "0.0.115"
+name = "letterbox-web"
-name = "letterbox"
+description = "Web frontend for letterbox"
-repository = "https://github.com/seed-rs/seed-quickstart"
+authors.workspace = true
-authors = ["Bill Thiede <git@xinu.tv>"]
+edition.workspace = true
-description = "App Description"
+license.workspace = true
-categories = ["category"]
+publish.workspace = true
-license = "MIT"
+repository.workspace = true
-readme = "./README.md"
+version.workspace = true
-edition = "2021"
 
 [build-dependencies]
-build-info-build = "0.0.38"
+build-info-build = "0.0.40"
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.33"
@@ -22,18 +21,22 @@ seed = { version = "0.10.0", features = ["routing"] }
 #seed = "0.9.2"
 console_log = { version = "0.1.0", registry = "xinu" }
 serde = { version = "1.0.147", features = ["derive"] }
-notmuch = { path = "../notmuch" }
+itertools = "0.14.0"
-shared = { path = "../shared" }
-itertools = "0.10.5"
 serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
 chrono = "0.4.31"
-graphql_client = "0.13.0"
+graphql_client = "0.14.0"
-thiserror = "1.0.50"
+thiserror = "2.0.0"
-seed_hooks = { git = "https://github.com/wathiede/styles_hooks", package = "seed_hooks", branch = "main" }
+gloo-net = { version = "0.6.0", features = ["json", "serde_json"] }
-gloo-net = { version = "0.4.0", features = ["json", "serde_json"] }
 human_format = "1.1.0"
-build-info = "0.0.38"
+build-info = "0.0.40"
-wasm-bindgen = "0.2.95"
+wasm-bindgen = "=0.2.100"
+uuid = { version = "1.13.1", features = [
+  "js",
+] } # direct dep to set js feature, prevents Rng issues
+letterbox-shared = { version = "0.10.0", path = "../shared", registry = "xinu" }
+letterbox-notmuch = { version = "0.10.0", path = "../notmuch", registry = "xinu" }
+seed_hooks = { version = "0.4.0", registry = "xinu" }
+strum_macros = "0.27.1"
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ['-Os']
@@ -47,4 +50,6 @@ features = [
   "MediaQueryList",
   "Navigator",
   "Window",
+  "History",
+  "ScrollRestoration",
 ]

web/graphql/catchup.graphql

@@ -0,0 +1,3 @@
+query CatchupQuery($query: String!) {
+    catchup(query: $query)
+}

web/graphql/schema.json

@@ -671,6 +671,30 @@
                 }
               }
             },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "xOriginalTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "deliveredTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
             {
               "args": [],
               "deprecationReason": null,
@@ -1268,6 +1292,45 @@
                 }
               }
             },
+            {
+              "args": [
+                {
+                  "defaultValue": null,
+                  "description": null,
+                  "name": "query",
+                  "type": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              ],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "catchup",
+              "type": {
+                "kind": "NON_NULL",
+                "name": null,
+                "ofType": {
+                  "kind": "LIST",
+                  "name": null,
+                  "ofType": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              }
+            },
             {
               "args": [
                 {

web/graphql/show_thread.graphql

@@ -31,6 +31,14 @@ query ShowThreadQuery($threadId: String!) {
           name
           addr
         }
+        xOriginalTo {
+          name
+          addr
+        }
+        deliveredTo {
+          name
+          addr
+        }
         timestamp
         body {
           __typename

web/index.html

@@ -4,8 +4,8 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.3.0/css/all.min.css"
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css"
-    integrity="sha512-SzlrxWUlpfuzQ+pcUCosxcglQRNAq/DZjVsC0lE40xsADsfeQoEypE+enwcOiGjk/bSuGGKHEyjSoQ1zVisanQ=="
+    integrity="sha512-Evv84Mr4kqVGRNSgIGL/F/aIDqQb7xQ2vcrdIwxfjThSH8CSR7PBEakCr51Ck+w+/U6swU2Im1vVX0SVk9ABhg=="
     crossorigin="anonymous" referrerpolicy="no-referrer" />
   <link rel="icon" href="https://static.xinu.tv/favicon/letterbox.svg" />
   <!-- tall thin font for user icon -->
@@ -13,7 +13,9 @@
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@700&display=swap" rel="stylesheet">
   <!-- <link data-trunk rel="css" href="static/site-specific.css" /> -->
+  <link data-trunk rel="css" href="static/vars.css" />
   <link data-trunk rel="tailwind-css" href="./src/tailwind.css" />
+  <link data-trunk rel="css" href="static/overrides.css" />
 </head>
 
 <body>

web/src/graphql.rs

@@ -12,6 +12,14 @@ use serde::{de::DeserializeOwned, Serialize};
 )]
 pub struct FrontPageQuery;
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/catchup.graphql",
+    response_derives = "Debug"
+)]
+pub struct CatchupQuery;
+
 #[derive(GraphQLQuery)]
 #[graphql(
     schema_path = "graphql/schema.json",

web/src/main.rs

@@ -18,6 +18,9 @@ fn main() {
     #[cfg(debug_assertions)]
     console_error_panic_hook::set_once();
 
+    #[cfg(debug_assertions)]
+    let lvl = Level::Debug;
+    #[cfg(not(debug_assertions))]
     let lvl = Level::Info;
     console_log::init_with_level(lvl).expect("failed to initialize console logging");
     // Mount the `app` to the element with the `id` "app".

web/src/state.rs

@@ -27,18 +27,23 @@ pub fn unread_query() -> &'static str {
 
 // `init` describes what should happen when your app started.
 pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
-    let version = shared::build_version(bi);
+    let version = letterbox_shared::build_version(bi);
     info!("Build Info: {}", version);
+    // Disable restoring to scroll position when navigating
+    window()
+        .history()
+        .expect("couldn't get history")
+        .set_scroll_restoration(web_sys::ScrollRestoration::Manual)
+        .expect("failed to set scroll restoration to manual");
     if url.hash().is_none() {
         orders.request_url(urls::search(unread_query(), 0));
     } else {
-        orders.notify(subs::UrlRequested::new(url));
+        orders.request_url(url);
     };
-    orders.stream(streams::window_event(Ev::Resize, |_| Msg::OnResize));
     // TODO(wathiede): only do this while viewing the index? Or maybe add a new message that force
     // 'notmuch new' on the server periodically?
     orders.stream(streams::interval(30_000, || Msg::RefreshStart));
-    orders.subscribe(on_url_changed);
+    orders.subscribe(Msg::OnUrlChanged);
     orders.stream(streams::window_event(Ev::Scroll, |_| Msg::WindowScrolled));
 
     build_info::build_info!(fn bi);
@@ -53,18 +58,23 @@ pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
             client: version,
             server: None,
         },
+        catchup: None,
+        last_url: Url::current(),
     }
 }
 
-fn on_url_changed(uc: subs::UrlChanged) -> Msg {
+fn on_url_changed(old: &Url, mut new: Url) -> Msg {
-    let mut url = uc.0;
+    let did_change = *old != new;
+    let mut messages = Vec::new();
+    if did_change {
+        messages.push(Msg::ScrollToTop)
+    }
     info!(
-        "url changed '{}', history {}",
+        "url changed\nold '{old}'\nnew '{new}', history {}",
-        url,
         history().length().unwrap_or(0)
     );
-    let hpp = url.remaining_hash_path_parts();
+    let hpp = new.remaining_hash_path_parts();
-    match hpp.as_slice() {
+    let msg = match hpp.as_slice() {
         ["t", tid] => Msg::ShowThreadRequest {
             thread_id: tid.to_string(),
         },
@@ -101,11 +111,14 @@ fn on_url_changed(uc: subs::UrlChanged) -> Msg {
                 last: None,
             }
         }
-    }
+    };
+    messages.push(msg);
+    Msg::MultiMsg(messages)
 }
 
 // `update` describes how to handle each `Msg`.
 pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
+    debug!("update({})", msg);
     match msg {
         Msg::Noop => {}
         Msg::RefreshStart => {
@@ -131,7 +144,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             orders.perform_cmd(async move { Msg::Refresh });
         }
         Msg::Refresh => {
-            orders.perform_cmd(async move { on_url_changed(subs::UrlChanged(Url::current())) });
+            orders.request_url(Url::current());
         }
         Msg::Reload => {
             window()
@@ -139,7 +152,10 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 .reload()
                 .expect("failed to reload window");
         }
-        Msg::OnResize => (),
+        Msg::OnUrlChanged(new_url) => {
+            orders.send_msg(on_url_changed(&model.last_url, new_url.0.clone()));
+            model.last_url = new_url.0;
+        }
 
         Msg::NextPage => {
             match &model.context {
@@ -181,10 +197,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             };
         }
         Msg::GoToSearchResults => {
-            let url = urls::search(&model.query, 0);
+            orders.send_msg(Msg::SearchQuery(model.query.clone()));
-            info!("GoToSearchRestuls Start");
-            orders.request_url(url);
-            info!("GoToSearchRestuls End");
         }
 
         Msg::UpdateQuery(query) => model.query = query,
@@ -279,12 +292,16 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 )
             });
         }
-        Msg::FrontPageResult(Err(e)) => error!("error FrontPageResult: {e:?}"),
+        Msg::FrontPageResult(Err(e)) => {
+            orders.send_msg(Msg::Reload);
+            error!("error FrontPageResult: {e:?}");
+        }
         Msg::FrontPageResult(Ok(graphql_client::Response {
             data: None,
             errors: None,
             ..
         })) => {
+            orders.send_msg(Msg::Reload);
             error!("FrontPageResult no data or errors, should not happen");
         }
         Msg::FrontPageResult(Ok(graphql_client::Response {
@@ -292,6 +309,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             errors: Some(e),
             ..
         })) => {
+            orders.send_msg(Msg::Reload);
             error!("FrontPageResult error: {e:?}");
         }
         Msg::FrontPageResult(Ok(graphql_client::Response {
@@ -307,7 +325,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     })
                     .collect(),
             );
-            info!("pager {:#?}", data.search.page_info);
             let selected_threads = 'context: {
                 if let Context::SearchResult {
                     results,
@@ -388,8 +405,42 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             orders.send_msg(Msg::WindowScrolled);
         }
         Msg::ShowThreadResult(bad) => {
+            orders.send_msg(Msg::Reload);
             error!("show_thread_query error: {bad:#?}");
         }
+        Msg::CatchupRequest { query } => {
+            orders.perform_cmd(async move {
+                Msg::CatchupResult(
+                    send_graphql::<_, graphql::catchup_query::ResponseData>(
+                        graphql::CatchupQuery::build_query(graphql::catchup_query::Variables {
+                            query,
+                        }),
+                    )
+                    .await,
+                )
+            });
+        }
+        Msg::CatchupResult(Ok(graphql_client::Response {
+            data: Some(data), ..
+        })) => {
+            let items = data.catchup;
+            if items.is_empty() {
+                orders.send_msg(Msg::GoToSearchResults);
+                model.catchup = None;
+            } else {
+                orders.request_url(urls::thread(&items[0]));
+                model.catchup = Some(Catchup {
+                    items: items
+                        .into_iter()
+                        .map(|id| CatchupItem { id, seen: false })
+                        .collect(),
+                });
+            }
+        }
+        Msg::CatchupResult(bad) => {
+            orders.send_msg(Msg::Reload);
+            error!("catchup_query error: {bad:#?}");
+        }
         Msg::SelectionSetNone => {
             if let Context::SearchResult {
                 selected_threads, ..
@@ -496,18 +547,19 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
         }
         Msg::MultiMsg(msgs) => msgs.into_iter().for_each(|msg| update(msg, model, orders)),
         Msg::CopyToClipboard(text) => {
-            let clipboard = seed::window()
+            let clipboard = seed::window().navigator().clipboard();
-                .navigator()
-                .clipboard()
-                .expect("couldn't get clipboard");
             orders.perform_cmd(async move {
                 wasm_bindgen_futures::JsFuture::from(clipboard.write_text(&text))
                     .await
                     .expect("failed to copy to clipboard");
             });
         }
+        Msg::ScrollToTop => {
+            info!("scrolling to the top");
+            web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
+        }
         Msg::WindowScrolled => {
-            info!("WindowScrolled");
+            // TODO: model.content_el doesn't go to None like it should when a DOM is recreated and the refrenced element goes away
             if let Some(el) = model.content_el.get() {
                 let ih = window()
                     .inner_height()
@@ -516,7 +568,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .value_of();
 
                 let r = el.get_bounding_client_rect();
-                info!("r {r:?} ih {ih}");
                 if r.height() < ih {
                     // The whole content fits in the window, no scrollbar
                     orders.send_msg(Msg::SetProgress(0.));
@@ -553,12 +604,82 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     "Server ({}) and client ({}) version mismatch, reloading",
                     version, model.versions.client
                 );
+                #[cfg(not(debug_assertions))]
                 orders.send_msg(Msg::Reload);
             }
             model.versions.server = Some(version);
         }
+
+        Msg::CatchupStart => {
+            let query = if model.query.contains("is:unread") {
+                model.query.to_string()
+            } else {
+                format!("{} is:unread", model.query)
+            };
+            info!("starting catchup mode w/ {}", query);
+            orders.send_msg(Msg::ScrollToTop);
+            orders.send_msg(Msg::CatchupRequest { query });
+        }
+        Msg::CatchupKeepUnread => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, true));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupMarkAsRead => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, false));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupNext => {
+            orders.send_msg(Msg::ScrollToTop);
+            let Some(catchup) = &mut model.catchup else {
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            let Some(idx) = catchup.items.iter().position(|i| !i.seen) else {
+                // All items have been seen
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            catchup.items[idx].seen = true;
+            if idx < catchup.items.len() - 1 {
+                // Reached last item
+                orders.request_url(urls::thread(&catchup.items[idx + 1].id));
+                return;
+            } else {
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+        }
+        Msg::CatchupExit => {
+            orders.send_msg(Msg::ScrollToTop);
+            model.catchup = None;
+        }
     }
 }
+
+fn current_thread_id(context: &Context) -> Option<String> {
+    match context {
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                    thread_id, ..
+                }),
+            ..
+        } => Some(thread_id.clone()),
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::NewsPost(ShowThreadQueryThreadOnNewsPost { thread_id, .. }),
+            ..
+        } => Some(thread_id.clone()),
+        _ => None,
+    }
+}
+
 // `Model` describes our app state.
 pub struct Model {
     pub query: String,
@@ -568,6 +689,8 @@ pub struct Model {
     pub read_completion_ratio: f64,
     pub content_el: ElRef<HtmlElement>,
     pub versions: Version,
+    pub catchup: Option<Catchup>,
+    pub last_url: Url,
 }
 
 #[derive(Debug)]
@@ -604,6 +727,15 @@ pub enum Context {
     },
 }
 
+pub struct Catchup {
+    pub items: Vec<CatchupItem>,
+}
+
+pub struct CatchupItem {
+    pub id: String,
+    pub seen: bool,
+}
+
 pub struct Tag {
     pub name: String,
     pub bg_color: String,
@@ -617,14 +749,15 @@ pub enum RefreshingState {
     Error(String),
 }
 // `Msg` describes the different events you can modify state with.
+#[derive(strum_macros::Display)]
 pub enum Msg {
     Noop,
     // Tell the client to refresh its state
     Refresh,
     // Tell the client to reload whole page from server
     Reload,
-    // Window has changed size
+    // TODO: add GoToUrl
-    OnResize,
+    OnUrlChanged(subs::UrlChanged),
     // Tell the server to update state
     RefreshStart,
     RefreshDone(Option<gloo_net::Error>),
@@ -654,12 +787,17 @@ pub enum Msg {
     ShowThreadResult(
         Result<graphql_client::Response<graphql::show_thread_query::ResponseData>, gloo_net::Error>,
     ),
+    CatchupRequest {
+        query: String,
+    },
+    CatchupResult(
+        Result<graphql_client::Response<graphql::catchup_query::ResponseData>, gloo_net::Error>,
+    ),
 
     SelectionSetNone,
     SelectionSetAll,
     SelectionAddTag(String),
     #[allow(dead_code)]
-    // TODO
     SelectionRemoveTag(String),
     SelectionMarkAsRead,
     SelectionMarkAsUnread,
@@ -672,7 +810,14 @@ pub enum Msg {
 
     CopyToClipboard(String),
 
+    ScrollToTop,
     WindowScrolled,
     SetProgress(f64),
     UpdateServerVersion(String),
+
+    CatchupStart,
+    CatchupKeepUnread,
+    CatchupMarkAsRead,
+    CatchupNext,
+    CatchupExit,
 }

web/static/overrides.css

@@ -0,0 +1,69 @@
+html {
+    background-color: black;
+}
+
+.mail-thread a,
+.news-post a {
+    color: var(--color-link) !important;
+    text-decoration: underline;
+}
+
+.mail-thread br,
+.news-post br {
+    display: block;
+    margin-top: 1em;
+    content: " ";
+}
+
+.mail-thread h1,
+.mail-thread h2,
+.mail-thread h3,
+.mail-thread h4,
+.news-post h1,
+.news-post h2,
+.news-post h3,
+.news-post h4 {
+    margin-top: 1em !important;
+    margin-bottom: 1em !important;
+}
+
+.mail-thread p,
+.news-post p {
+    margin-bottom: 1em;
+}
+
+.mail-thread pre,
+.mail-thread code,
+.news-post pre,
+.news-post code {
+    font-family: monospace;
+    background-color: #eee !important;
+    padding: 0.5em !important;
+}
+
+.mail-thread blockquote {
+    padding-left: 1em;
+    border-left: 2px solid #ddd;
+}
+
+.mail-thread ol,
+.mail-thread ul {
+    margin-left: 2em;
+}
+
+/* Hackaday figures have unreadable black on dark grey */
+.news-post figcaption.wp-caption-text {
+    background-color: initial !important;
+}
+
+.news-post.site-nautilus .article-ad,
+.news-post.site-nautilus .primis-ad {
+    display: none !important;
+}
+
+.news-post.site-slashdot .story-byline {
+    display: block !important;
+    height: initial !important;
+    overflow: auto !important;
+    position: static !important;
+}

web/static/vars.css

@@ -0,0 +1,42 @@
+:root {
+    --active-brightness: 0.85;
+    --border-radius: 5px;
+    --box-shadow: 2px 2px 10px;
+    --color-accent: #118bee15;
+    --color-bg: #fff;
+    --color-bg-secondary: #e9e9e9;
+    --color-link: #118bee;
+    --color-secondary: #920de9;
+    --color-secondary-accent: #920de90b;
+    --color-shadow: #f4f4f4;
+    --color-table: #118bee;
+    --color-text: #000;
+    --color-text-secondary: #999;
+    --color-scrollbar: #cacae8;
+    --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+    --hover-brightness: 1.2;
+    --justify-important: center;
+    --justify-normal: left;
+    --line-height: 1.5;
+    /*
+  --width-card: 285px;
+  --width-card-medium: 460px;
+  --width-card-wide: 800px;
+*/
+    --width-content: 1080px;
+}
+
+@media (prefers-color-scheme: dark) {
+    :root[color-mode="user"] {
+        --color-accent: #0097fc4f;
+        --color-bg: #333;
+        --color-bg-secondary: #555;
+        --color-link: #0097fc;
+        --color-secondary: #e20de9;
+        --color-secondary-accent: #e20de94f;
+        --color-shadow: #bbbbbb20;
+        --color-table: #0097fc;
+        --color-text: #f7f7f7;
+        --color-text-secondary: #aaa;
+    }
+}