chore: Release

This reverts commit 1411961e36.

.gitea/workflows/rust.yml

@@ -1,4 +1,4 @@
-on: [push, pull_request]
+on: [push]
 
 name: Continuous integration
 
@@ -49,3 +49,19 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
       - run: cargo build
+
+  udeps:
+    name: Disallow unused dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+
+      - name: Run cargo-udeps
+        uses: aig787/cargo-udeps-action@v1
+        with:
+          version: 'latest'
+          args: '--all-targets'
+

Cargo.toml

@@ -1,8 +1,15 @@
 [workspace]
 resolver = "2"
-#default-members = ["server"]
-members = ["notmuch", "procmail2notmuch", "shared"]
-#members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+default-members = ["server"]
+members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+
+[workspace.package]
+authors = ["Bill Thiede <git@xinu.tv>"]
+edition = "2021"
+license = "UNLICENSED"
+publish = ["xinu"]
+version = "0.12.1"
+repository = "https://git.z.xinu.tv/wathiede/letterbox"
 
 [profile.dev]
 opt-level = 1

Justfile

@@ -0,0 +1,19 @@
+export CARGO_INCREMENTAL := "0"
+export RUSTFLAGS := "-D warnings"
+
+default:
+    @echo "Run: just patch|minor|major"
+
+major: (_release "major")
+minor: (_release "minor")
+patch: (_release "patch")
+
+sqlx-prepare:
+	cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true
+
+pull:
+    git pull
+
+
+_release level: pull sqlx-prepare
+    cargo-release release  -x {{ level }} --workspace --no-confirm --registry=xinu

notmuch/Cargo.toml

@@ -1,16 +1,18 @@
 [package]
 name = "letterbox-notmuch"
-version = "0.1.0"
-edition = "2021"
 exclude = ["/testdata"]
 description = "Wrapper for calling notmuch cli"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 
 [dependencies]
 log = "0.4.14"
+mailparse = "0.16.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
 thiserror = "2.0.0"

notmuch/src/lib.rs

@@ -207,6 +207,7 @@
 //! ```
 
 use std::{
+    collections::HashMap,
     ffi::OsStr,
     io::{self},
     path::{Path, PathBuf},
@@ -270,6 +271,12 @@ pub struct Headers {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub bcc: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "Delivered-To")]
+    pub delivered_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "X-Original-To")]
+    pub x_original_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub reply_to: Option<String>,
     pub date: String,
 }
@@ -459,6 +466,8 @@ pub enum NotmuchError {
     StringUtf8Error(#[from] std::string::FromUtf8Error),
     #[error("failed to parse str as int")]
     ParseIntError(#[from] std::num::ParseIntError),
+    #[error("failed to parse mail: {0}")]
+    MailParseError(#[from] mailparse::MailParseError),
 }
 
 #[derive(Default)]
@@ -605,6 +614,80 @@ impl Notmuch {
         Ok(serde_json::from_slice(&res)?)
     }
 
+    #[instrument(skip_all)]
+    pub fn unread_recipients(&self) -> Result<HashMap<String, usize>, NotmuchError> {
+        let slice = self.run_notmuch([
+            "show",
+            "--include-html=false",
+            "--entire-thread=false",
+            "--body=false",
+            "--format=json",
+            // Arbitrary limit to prevent too much work
+            "--limit=1000",
+            "is:unread",
+        ])?;
+        // Notmuch returns JSON with invalid unicode.  So we lossy convert it to a string here and
+        // use that for parsing in rust.
+        let s = String::from_utf8_lossy(&slice);
+        let mut deserializer = serde_json::Deserializer::from_str(&s);
+        deserializer.disable_recursion_limit();
+        let ts: ThreadSet = serde::de::Deserialize::deserialize(&mut deserializer)?;
+        deserializer.end()?;
+        let mut r = HashMap::new();
+        fn collect_from_thread_node(
+            r: &mut HashMap<String, usize>,
+            tn: &ThreadNode,
+        ) -> Result<(), NotmuchError> {
+            let Some(msg) = &tn.0 else {
+                return Ok(());
+            };
+            let mut addrs = vec![];
+            let hdr = &msg.headers.to;
+            if let Some(to) = hdr {
+                addrs.push(to);
+            } else {
+                let hdr = &msg.headers.x_original_to;
+                if let Some(to) = hdr {
+                    addrs.push(to);
+                } else {
+                    let hdr = &msg.headers.delivered_to;
+                    if let Some(to) = hdr {
+                        addrs.push(to);
+                    };
+                };
+            };
+            let hdr = &msg.headers.cc;
+            if let Some(cc) = hdr {
+                addrs.push(cc);
+            };
+            for recipient in addrs {
+                mailparse::addrparse(&recipient)?
+                    .into_inner()
+                    .iter()
+                    .for_each(|a| {
+                        let mailparse::MailAddr::Single(si) = a else {
+                            return;
+                        };
+                        let addr = &si.addr;
+
+                        if addr == "couchmoney@gmail.com" || addr.ends_with("@xinu.tv") {
+                            *r.entry(addr.to_lowercase()).or_default() += 1;
+                        }
+                    });
+            }
+            Ok(())
+        }
+        for t in ts.0 {
+            for tn in t.0 {
+                collect_from_thread_node(&mut r, &tn)?;
+                for sub_tn in tn.1 {
+                    collect_from_thread_node(&mut r, &sub_tn)?;
+                }
+            }
+        }
+        Ok(r)
+    }
+
     fn run_notmuch<I, S>(&self, args: I) -> Result<Vec<u8>, NotmuchError>
     where
         I: IntoIterator<Item = S>,

procmail2notmuch/Cargo.toml

@@ -1,11 +1,12 @@
 [package]
 name = "letterbox-procmail2notmuch"
-version = "0.1.0"
-edition = "2021"
 description = "Tool for generating notmuch rules from procmail"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

server/Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "letterbox-server"
-version = "0.1.0"
-edition = "2021"
 default-run = "letterbox-server"
 description = "Backend for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -16,8 +17,8 @@ anyhow = "1.0.79"
 async-graphql = { version = "7", features = ["log"] }
 async-graphql-rocket = "7"
 async-trait = "0.1.81"
-build-info = "0.0.39"
-cacher = { version = "0.1.0", registry = "xinu" }
+build-info = "0.0.40"
+cacher = { version = "0.2.0", registry = "xinu" }
 chrono = "0.4.39"
 clap = { version = "4.5.23", features = ["derive"] }
 css-inline = "0.14.0"
@@ -29,16 +30,15 @@ lol_html = "2.0.0"
 mailparse = "0.16.0"
 maplit = "1.0.2"
 memmap = "0.7.0"
-opentelemetry = "0.28.0"
 regex = "1.11.1"
 reqwest = { version = "0.12.7", features = ["blocking"] }
 rocket = { version = "0.5.0-rc.2", features = ["json"] }
 rocket_cors = "0.6.0"
-scraper = "0.22.0"
+scraper = "0.23.0"
 serde = { version = "1.0.147", features = ["derive"] }
 serde_json = "1.0.87"
 sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
-tantivy = { version = "0.22.0", optional = true }
+tantivy = { version = "0.24.0", optional = true }
 thiserror = "2.0.0"
 tokio = "1.26.0"
 tracing = "0.1.41"
@@ -46,12 +46,12 @@ url = "2.5.2"
 urlencoding = "2.1.3"
 #xtracing = { path = "../../xtracing" }
 #xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
-xtracing = { version = "0.2.0", registry = "xinu" }
-letterbox-notmuch = { version = "0.1.0", registry = "xinu" }
-letterbox-shared = { version = "0.1.0", registry = "xinu" }
+xtracing = { version = "0.3.0", registry = "xinu" }
+letterbox-notmuch = { version = "0.12.1", path = "../notmuch", registry = "xinu" }
+letterbox-shared = { version = "0.12.1", path = "../shared", registry = "xinu" }
 
 [build-dependencies]
-build-info-build = "0.0.39"
+build-info-build = "0.0.40"
 
 [features]
 #default = [ "tantivy" ]

server/migrations/20250330175930_update-nzfinder-link.sql

@@ -0,0 +1,20 @@
+-- Bad examples:
+ -- https://nzbfinder.ws/getnzb/d2c3e5a08abadd985dccc6a574122892030b6a9a.nzb&i=95972&r=b55082d289937c050dedc203c9653850
+--  https://nzbfinder.ws/getnzb?id=45add174-7da4-4445-bf2b-a67dbbfc07fe.nzb&r=b55082d289937c050dedc203c9653850
+--  https://nzbfinder.ws/api/v1/getnzb?id=82486020-c192-4fa0-a7e7-798d7d72e973.nzb&r=b55082d289937c050dedc203c9653850
+UPDATE nzb_posts
+SET link = 
+regexp_replace(
+    regexp_replace(
+        regexp_replace(
+            link,
+            'https://nzbfinder.ws/getnzb/',
+            'https://nzbfinder.ws/api/v1/getnzb?id='
+            ),
+        'https://nzbfinder.ws/getnzb',
+        'https://nzbfinder.ws/api/v1/getnzb'
+        ),
+    '&r=',
+    '&apikey='
+    ) 
+;

server/src/graphql.rs

@@ -95,6 +95,10 @@ pub struct Message {
     pub to: Vec<Email>,
     // All CC headers found in email
     pub cc: Vec<Email>,
+    // X-Original-To header found in email
+    pub x_original_to: Option<Email>,
+    // Delivered-To header found in email
+    pub delivered_to: Option<Email>,
     // First Subject header found in email
     pub subject: Option<String>,
     // Parsed Date header, if found and valid
@@ -286,7 +290,7 @@ impl QueryRoot {
         Ok(letterbox_shared::build_version(bi))
     }
     #[instrument(skip_all, fields(query=query))]
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn count<'ctx>(&self, ctx: &Context<'ctx>, query: String) -> Result<usize, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -306,10 +310,49 @@ impl QueryRoot {
         info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count} tantivy count {tantivy_count} total {total}");
         Ok(total)
     }
+    async fn catchup<'ctx>(
+        &self,
+        ctx: &Context<'ctx>,
+        query: String,
+    ) -> Result<Vec<String>, Error> {
+        let nm = ctx.data_unchecked::<Notmuch>();
+        let pool = ctx.data_unchecked::<PgPool>();
+        let query: Query = query.parse()?;
+        // TODO: implement optimized versions of fetching just IDs
+        let newsreader_fut = newsreader_search(pool, None, None, None, None, &query);
+        let notmuch_fut = notmuch_search(nm, None, None, None, None, &query);
+        let (newsreader_results, notmuch_results) = join!(newsreader_fut, notmuch_fut);
+
+        let newsreader_results = newsreader_results?;
+        let notmuch_results = notmuch_results?;
+        info!(
+            "newsreader_results ({}) notmuch_results ({})",
+            newsreader_results.len(),
+            notmuch_results.len(),
+        );
+
+        let mut results: Vec<_> = newsreader_results
+            .into_iter()
+            .chain(notmuch_results)
+            .collect();
+        // The leading '-' is to reverse sort
+        results.sort_by_key(|item| match item {
+            ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
+            ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
+        });
+        let ids = results
+            .into_iter()
+            .map(|r| match r {
+                ThreadSummaryCursor::Newsreader(_, ts) => ts.thread,
+                ThreadSummaryCursor::Notmuch(_, ts) => ts.thread,
+            })
+            .collect();
+        Ok(ids)
+    }
 
     // TODO: this function doesn't get parallelism, possibly because notmuch is sync and blocks,
     // rewrite that with tokio::process:Command
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn search<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -467,7 +510,7 @@ impl QueryRoot {
         .await?)
     }
 
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -476,7 +519,7 @@ impl QueryRoot {
         tags.append(&mut nm::tags(nm, needs_unread)?);
         Ok(tags)
     }
-    #[instrument(skip_all, fields(thread_id=thread_id, request_id=request_id()))]
+    #[instrument(skip_all, fields(thread_id=thread_id, rid=request_id()))]
     async fn thread<'ctx>(&self, ctx: &Context<'ctx>, thread_id: String) -> Result<Thread, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let cacher = ctx.data_unchecked::<FilesystemCacher>();
@@ -553,7 +596,7 @@ async fn tantivy_search(
 pub struct Mutation;
 #[Object]
 impl Mutation {
-    #[instrument(skip_all, fields(query=query, unread=unread, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, unread=unread, rid=request_id()))]
     async fn set_read_status<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -572,7 +615,7 @@ impl Mutation {
         nm::set_read_status(nm, &query, unread).await?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_add<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -584,7 +627,7 @@ impl Mutation {
         nm.tag_add(&tag, &query)?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_remove<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -607,7 +650,7 @@ impl Mutation {
 
         Ok(true)
     }
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn refresh<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let cacher = ctx.data_unchecked::<FilesystemCacher>();

server/src/lib.rs

@@ -318,13 +318,16 @@ impl<'c> Transformer for SlurpContents<'c> {
         } else {
             let resp = reqwest::get(link.as_str()).await?;
             let status = resp.status();
-            if status.is_server_error() || retryable_status.contains(&status) {
-                return Err(TransformError::RetryableHttpStatusError(
-                    status,
-                    link.to_string(),
-                ));
+            if status.is_server_error() {
+                error!("status error for {link}: {status}");
+                return Ok(html.to_string());
+            }
+            if retryable_status.contains(&status) {
+                error!("retryable error for {link}: {status}");
+                return Ok(html.to_string());
             }
             if !status.is_success() {
+                error!("unsuccessful for {link}: {status}");
                 return Ok(html.to_string());
             }
             let body = resp.text().await?;
@@ -341,7 +344,7 @@ impl<'c> Transformer for SlurpContents<'c> {
                     "/* chrome-default.css */\n",
                     include_str!("chrome-default.css"),
                     "\n/* vars.css */\n",
-                    include_str!("../../web/static/vars.css"),
+                    include_str!("../static/vars.css"),
                     //"\n/* Xinu Specific overrides */\n",
                     //include_str!("custom.css"),
                 );
@@ -438,6 +441,38 @@ pub fn sanitize_html(
         }
     };
     let mut element_content_handlers = vec![
+        // Remove width and height attributes on elements
+        element!("[width],[height]", |el| {
+            println!("width or height {el:?}");
+            el.remove_attribute("width");
+            el.remove_attribute("height");
+            Ok(())
+        }),
+        // Remove width and height values from inline styles
+        element!("[style]", |el| {
+            println!("style {el:?}");
+            let style = el.get_attribute("style").unwrap();
+            let style = style
+                .split(";")
+                .filter(|s| {
+                    println!("s {s}");
+                    let Some((k, _)) = s.split_once(':') else {
+                        return true;
+                    };
+                    match k {
+                        "width" | "max-width" | "min-width" | "height" | "max-height"
+                        | "min-height" => false,
+                        _ => true,
+                    }
+                })
+                .collect::<Vec<_>>()
+                .join(";");
+            println!("style: {style}");
+            if let Err(e) = el.set_attribute("style", &style) {
+                error!("Failed to set style attribute: {e}");
+            }
+            Ok(())
+        }),
         // Open links in new tab
         element!("a[href]", |el| {
             el.set_attribute("target", "_blank").unwrap();
@@ -682,7 +717,7 @@ fn compute_offset_limit(
     first: Option<i32>,
     last: Option<i32>,
 ) -> (i32, i32) {
-    let default_page_size = 100;
+    let default_page_size = 10000;
     match (after, before, first, last) {
         // Reasonable defaults
         (None, None, None, None) => (0, default_page_size),
@@ -773,7 +808,19 @@ impl Query {
         for uid in &self.uids {
             parts.push(uid.clone());
         }
-        parts.extend(self.remainder.clone());
+        for r in &self.remainder {
+            // Rewrite "to:" to include ExtraTo:. ExtraTo: is configured in
+            // notmuch-config to index Delivered-To and X-Original-To headers.
+            if r.starts_with("to:") {
+                parts.push("(".to_string());
+                parts.push(r.to_string());
+                parts.push("OR".to_string());
+                parts.push(r.replace("to:", "ExtraTo:"));
+                parts.push(")".to_string());
+            } else {
+                parts.push(r.to_string());
+            }
+        }
         parts.join(" ")
     }
 }
@@ -793,7 +840,17 @@ impl FromStr for Query {
             if word == "is:unread" {
                 unread_only = true
             } else if word.starts_with("tag:") {
-                tags.push(word["tag:".len()..].to_string());
+                let t = &word["tag:".len()..];
+                // Per-address emails are faked as `tag:@<domain>/<username>`, rewrite to `to:` form
+                if t.starts_with('@') && t.contains('.') {
+                    let t = match t.split_once('/') {
+                        None => format!("to:{t}"),
+                        Some((domain, user)) => format!("to:{user}{domain}"),
+                    };
+                    remainder.push(t);
+                } else {
+                    tags.push(t.to_string());
+                };
 
                 /*
                             } else if word.starts_with("tag:") {
@@ -888,3 +945,21 @@ async fn clean_title(title: &str) -> Result<String, ServerError> {
     }
     Ok(title)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{SanitizeHtml, Transformer};
+
+    #[tokio::test]
+    async fn strip_sizes() -> Result<(), Box<dyn std::error::Error>> {
+        let ss = SanitizeHtml {
+            cid_prefix: "",
+            base_url: &None,
+        };
+        let input = r#"<p width=16 height=16 style="color:blue;width:16px;height:16px;">This el has width and height attributes and inline styles</p>"#;
+        let want = r#"<p style="color:blue;">This el has width and height attributes and inline styles</p>"#;
+        let got = ss.transform(&None, input).await?;
+        assert_eq!(got, want);
+        Ok(())
+    }
+}

server/src/nm.rs

@@ -1,11 +1,7 @@
-use std::{
-    collections::HashMap,
-    fs::File,
-    hash::{DefaultHasher, Hash, Hasher},
-    time::Instant,
-};
+use std::{collections::HashMap, fs::File};
 
 use letterbox_notmuch::Notmuch;
+use letterbox_shared::compute_color;
 use log::{error, info, warn};
 use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
 use memmap::MmapOptions;
@@ -107,7 +103,6 @@ pub async fn search(
 
 #[instrument(name="nm::tags", skip_all, fields(needs_unread=needs_unread))]
 pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
-    let now = Instant::now();
     let unread_msg_cnt: HashMap<String, usize> = if needs_unread {
         // 10000 is an arbitrary number, if there's more than 10k unread messages, we'll
         //  get an inaccurate count.
@@ -123,13 +118,11 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     } else {
         HashMap::new()
     };
-    let tags = nm
+    let tags: Vec<_> = nm
         .tags()?
         .into_iter()
         .map(|tag| {
-            let mut hasher = DefaultHasher::new();
-            tag.hash(&mut hasher);
-            let hex = format!("#{:06x}", hasher.finish() % (1 << 24));
+            let hex = compute_color(&tag);
             let unread = if needs_unread {
                 *unread_msg_cnt.get(&tag).unwrap_or(&0)
             } else {
@@ -142,8 +135,24 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
                 unread,
             }
         })
+        .chain(
+            nm.unread_recipients()?
+                .into_iter()
+                .filter_map(|(name, unread)| {
+                    let Some(idx) = name.find('@') else {
+                        return None;
+                    };
+                    let name = format!("{}/{}", &name[idx..], &name[..idx]);
+                    let bg_color = compute_color(&name);
+                    Some(Tag {
+                        name,
+                        fg_color: "white".to_string(),
+                        bg_color,
+                        unread,
+                    })
+                }),
+        )
         .collect();
-    info!("Fetching tags took {} seconds", now.elapsed().as_secs_f32());
     Ok(tags)
 }
 
@@ -187,6 +196,8 @@ pub async fn thread(
 
         let to = email_addresses(&path, &m, "to")?;
         let cc = email_addresses(&path, &m, "cc")?;
+        let delivered_to = email_addresses(&path, &m, "delivered-to")?.pop();
+        let x_original_to = email_addresses(&path, &m, "x-original-to")?.pop();
         let subject = m.headers.get_first_value("subject");
         let timestamp = m
             .headers
@@ -306,6 +317,8 @@ pub async fn thread(
             body,
             path,
             attachments,
+            delivered_to,
+            x_original_to,
         });
     }
     messages.reverse();

server/static/vars.css

@@ -0,0 +1,42 @@
+:root {
+    --active-brightness: 0.85;
+    --border-radius: 5px;
+    --box-shadow: 2px 2px 10px;
+    --color-accent: #118bee15;
+    --color-bg: #fff;
+    --color-bg-secondary: #e9e9e9;
+    --color-link: #118bee;
+    --color-secondary: #920de9;
+    --color-secondary-accent: #920de90b;
+    --color-shadow: #f4f4f4;
+    --color-table: #118bee;
+    --color-text: #000;
+    --color-text-secondary: #999;
+    --color-scrollbar: #cacae8;
+    --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+    --hover-brightness: 1.2;
+    --justify-important: center;
+    --justify-normal: left;
+    --line-height: 1.5;
+    /*
+  --width-card: 285px;
+  --width-card-medium: 460px;
+  --width-card-wide: 800px;
+*/
+    --width-content: 1080px;
+}
+
+@media (prefers-color-scheme: dark) {
+    :root[color-mode="user"] {
+        --color-accent: #0097fc4f;
+        --color-bg: #333;
+        --color-bg-secondary: #555;
+        --color-link: #0097fc;
+        --color-secondary: #e20de9;
+        --color-secondary-accent: #e20de94f;
+        --color-shadow: #bbbbbb20;
+        --color-table: #0097fc;
+        --color-text: #f7f7f7;
+        --color-text-secondary: #aaa;
+    }
+}

shared/Cargo.toml

@@ -1,15 +1,16 @@
 [package]
 name = "letterbox-shared"
-version = "0.1.0"
-edition = "2021"
 description = "Shared module for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-build-info = "0.0.39"
-letterbox-notmuch = { version = "0.1.0", registry = "xinu" }
+build-info = "0.0.40"
+letterbox-notmuch = { version = "0.12.1", path = "../notmuch", registry = "xinu" }
 serde = { version = "1.0.147", features = ["derive"] }

web/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
-version = "0.1.0"
 name = "letterbox-web"
-authors = ["Bill Thiede <git@xinu.tv>"]
-edition = "2021"
 description = "Web frontend for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 [build-dependencies]
-build-info-build = "0.0.39"
+build-info-build = "0.0.40"
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.33"
@@ -26,16 +26,17 @@ serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
 chrono = "0.4.31"
 graphql_client = "0.14.0"
 thiserror = "2.0.0"
-seed_hooks = { git = "https://github.com/wathiede/styles_hooks", package = "seed_hooks", branch = "main" }
 gloo-net = { version = "0.6.0", features = ["json", "serde_json"] }
 human_format = "1.1.0"
-build-info = "0.0.39"
+build-info = "0.0.40"
 wasm-bindgen = "=0.2.100"
 uuid = { version = "1.13.1", features = [
   "js",
 ] } # direct dep to set js feature, prevents Rng issues
-letterbox-shared = { version = "0.1.0", registry = "xinu" }
-letterbox-notmuch = { version = "0.1.0", path = "../notmuch", registry = "xinu" }
+letterbox-shared = { version = "0.12.1", path = "../shared", registry = "xinu" }
+letterbox-notmuch = { version = "0.12.1", path = "../notmuch", registry = "xinu" }
+seed_hooks = { version = "0.4.0", registry = "xinu" }
+strum_macros = "0.27.1"
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ['-Os']
@@ -49,4 +50,6 @@ features = [
   "MediaQueryList",
   "Navigator",
   "Window",
+  "History",
+  "ScrollRestoration",
 ]

web/graphql/catchup.graphql

@@ -0,0 +1,3 @@
+query CatchupQuery($query: String!) {
+    catchup(query: $query)
+}

web/graphql/schema.json

@@ -671,6 +671,30 @@
                 }
               }
             },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "xOriginalTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "deliveredTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
             {
               "args": [],
               "deprecationReason": null,
@@ -1268,6 +1292,45 @@
                 }
               }
             },
+            {
+              "args": [
+                {
+                  "defaultValue": null,
+                  "description": null,
+                  "name": "query",
+                  "type": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              ],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "catchup",
+              "type": {
+                "kind": "NON_NULL",
+                "name": null,
+                "ofType": {
+                  "kind": "LIST",
+                  "name": null,
+                  "ofType": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              }
+            },
             {
               "args": [
                 {

web/graphql/show_thread.graphql

@@ -31,6 +31,14 @@ query ShowThreadQuery($threadId: String!) {
           name
           addr
         }
+        xOriginalTo {
+          name
+          addr
+        }
+        deliveredTo {
+          name
+          addr
+        }
         timestamp
         body {
           __typename

web/src/graphql.rs

@@ -12,6 +12,14 @@ use serde::{de::DeserializeOwned, Serialize};
 )]
 pub struct FrontPageQuery;
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/catchup.graphql",
+    response_derives = "Debug"
+)]
+pub struct CatchupQuery;
+
 #[derive(GraphQLQuery)]
 #[graphql(
     schema_path = "graphql/schema.json",

web/src/main.rs

@@ -18,6 +18,9 @@ fn main() {
     #[cfg(debug_assertions)]
     console_error_panic_hook::set_once();
 
+    #[cfg(debug_assertions)]
+    let lvl = Level::Debug;
+    #[cfg(not(debug_assertions))]
     let lvl = Level::Info;
     console_log::init_with_level(lvl).expect("failed to initialize console logging");
     // Mount the `app` to the element with the `id` "app".

web/src/state.rs

@@ -29,16 +29,21 @@ pub fn unread_query() -> &'static str {
 pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
     let version = letterbox_shared::build_version(bi);
     info!("Build Info: {}", version);
+    // Disable restoring to scroll position when navigating
+    window()
+        .history()
+        .expect("couldn't get history")
+        .set_scroll_restoration(web_sys::ScrollRestoration::Manual)
+        .expect("failed to set scroll restoration to manual");
     if url.hash().is_none() {
         orders.request_url(urls::search(unread_query(), 0));
     } else {
-        orders.notify(subs::UrlRequested::new(url));
+        orders.request_url(url);
     };
-    orders.stream(streams::window_event(Ev::Resize, |_| Msg::OnResize));
     // TODO(wathiede): only do this while viewing the index? Or maybe add a new message that force
     // 'notmuch new' on the server periodically?
     orders.stream(streams::interval(30_000, || Msg::RefreshStart));
-    orders.subscribe(on_url_changed);
+    orders.subscribe(Msg::OnUrlChanged);
     orders.stream(streams::window_event(Ev::Scroll, |_| Msg::WindowScrolled));
 
     build_info::build_info!(fn bi);
@@ -53,18 +58,23 @@ pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
             client: version,
             server: None,
         },
+        catchup: None,
+        last_url: Url::current(),
     }
 }
 
-fn on_url_changed(uc: subs::UrlChanged) -> Msg {
-    let mut url = uc.0;
+fn on_url_changed(old: &Url, mut new: Url) -> Msg {
+    let did_change = *old != new;
+    let mut messages = Vec::new();
+    if did_change {
+        messages.push(Msg::ScrollToTop)
+    }
     info!(
-        "url changed '{}', history {}",
-        url,
+        "url changed\nold '{old}'\nnew '{new}', history {}",
         history().length().unwrap_or(0)
     );
-    let hpp = url.remaining_hash_path_parts();
-    match hpp.as_slice() {
+    let hpp = new.remaining_hash_path_parts();
+    let msg = match hpp.as_slice() {
         ["t", tid] => Msg::ShowThreadRequest {
             thread_id: tid.to_string(),
         },
@@ -101,11 +111,14 @@ fn on_url_changed(uc: subs::UrlChanged) -> Msg {
                 last: None,
             }
         }
-    }
+    };
+    messages.push(msg);
+    Msg::MultiMsg(messages)
 }
 
 // `update` describes how to handle each `Msg`.
 pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
+    info!("update({})", msg);
     match msg {
         Msg::Noop => {}
         Msg::RefreshStart => {
@@ -131,7 +144,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             orders.perform_cmd(async move { Msg::Refresh });
         }
         Msg::Refresh => {
-            orders.perform_cmd(async move { on_url_changed(subs::UrlChanged(Url::current())) });
+            orders.request_url(Url::current());
         }
         Msg::Reload => {
             window()
@@ -139,7 +152,10 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 .reload()
                 .expect("failed to reload window");
         }
-        Msg::OnResize => (),
+        Msg::OnUrlChanged(new_url) => {
+            orders.send_msg(on_url_changed(&model.last_url, new_url.0.clone()));
+            model.last_url = new_url.0;
+        }
 
         Msg::NextPage => {
             match &model.context {
@@ -181,10 +197,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             };
         }
         Msg::GoToSearchResults => {
-            let url = urls::search(&model.query, 0);
-            info!("GoToSearchRestuls Start");
-            orders.request_url(url);
-            info!("GoToSearchRestuls End");
+            orders.send_msg(Msg::SearchQuery(model.query.clone()));
         }
 
         Msg::UpdateQuery(query) => model.query = query,
@@ -279,7 +292,9 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 )
             });
         }
-        Msg::FrontPageResult(Err(e)) => error!("error FrontPageResult: {e:?}"),
+        Msg::FrontPageResult(Err(e)) => {
+            error!("error FrontPageResult: {e:?}");
+        }
         Msg::FrontPageResult(Ok(graphql_client::Response {
             data: None,
             errors: None,
@@ -307,7 +322,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     })
                     .collect(),
             );
-            info!("pager {:#?}", data.search.page_info);
             let selected_threads = 'context: {
                 if let Context::SearchResult {
                     results,
@@ -390,6 +404,38 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
         Msg::ShowThreadResult(bad) => {
             error!("show_thread_query error: {bad:#?}");
         }
+        Msg::CatchupRequest { query } => {
+            orders.perform_cmd(async move {
+                Msg::CatchupResult(
+                    send_graphql::<_, graphql::catchup_query::ResponseData>(
+                        graphql::CatchupQuery::build_query(graphql::catchup_query::Variables {
+                            query,
+                        }),
+                    )
+                    .await,
+                )
+            });
+        }
+        Msg::CatchupResult(Ok(graphql_client::Response {
+            data: Some(data), ..
+        })) => {
+            let items = data.catchup;
+            if items.is_empty() {
+                orders.send_msg(Msg::GoToSearchResults);
+                model.catchup = None;
+            } else {
+                orders.request_url(urls::thread(&items[0]));
+                model.catchup = Some(Catchup {
+                    items: items
+                        .into_iter()
+                        .map(|id| CatchupItem { id, seen: false })
+                        .collect(),
+                });
+            }
+        }
+        Msg::CatchupResult(bad) => {
+            error!("catchup_query error: {bad:#?}");
+        }
         Msg::SelectionSetNone => {
             if let Context::SearchResult {
                 selected_threads, ..
@@ -503,8 +549,12 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .expect("failed to copy to clipboard");
             });
         }
+        Msg::ScrollToTop => {
+            info!("scrolling to the top");
+            web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
+        }
         Msg::WindowScrolled => {
-            info!("WindowScrolled");
+            // TODO: model.content_el doesn't go to None like it should when a DOM is recreated and the refrenced element goes away
             if let Some(el) = model.content_el.get() {
                 let ih = window()
                     .inner_height()
@@ -513,7 +563,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .value_of();
 
                 let r = el.get_bounding_client_rect();
-                info!("r {r:?} ih {ih}");
                 if r.height() < ih {
                     // The whole content fits in the window, no scrollbar
                     orders.send_msg(Msg::SetProgress(0.));
@@ -545,7 +594,8 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             model.read_completion_ratio = ratio;
         }
         Msg::UpdateServerVersion(version) => {
-            if version != model.versions.client {
+            // Only git versions contain dash, don't autoreload there
+            if !version.contains('-') && version != model.versions.client {
                 warn!(
                     "Server ({}) and client ({}) version mismatch, reloading",
                     version, model.versions.client
@@ -554,8 +604,82 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             }
             model.versions.server = Some(version);
         }
+
+        Msg::CatchupStart => {
+            let query = if model.query.contains("is:unread") {
+                model.query.to_string()
+            } else {
+                format!("{} is:unread", model.query)
+            };
+            info!("starting catchup mode w/ {}", query);
+            orders.send_msg(Msg::ScrollToTop);
+            orders.send_msg(Msg::CatchupRequest { query });
+        }
+        Msg::CatchupKeepUnread => {
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupMarkAsRead => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, false));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupNext => {
+            orders.send_msg(Msg::ScrollToTop);
+            let Some(catchup) = &mut model.catchup else {
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            let Some(thread_id) = current_thread_id(&model.context) else {
+                return;
+            };
+            let Some(idx) = catchup
+                .items
+                .iter()
+                .inspect(|i| info!("i {i:?} thread_id {thread_id}"))
+                .position(|i| i.id == thread_id)
+            else {
+                // All items have been seen
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            catchup.items[idx].seen = true;
+            if idx < catchup.items.len() - 1 {
+                // Reached last item
+                orders.request_url(urls::thread(&catchup.items[idx + 1].id));
+                return;
+            } else {
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+        }
+        Msg::CatchupExit => {
+            orders.send_msg(Msg::ScrollToTop);
+            model.catchup = None;
+        }
     }
 }
+
+fn current_thread_id(context: &Context) -> Option<String> {
+    match context {
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                    thread_id, ..
+                }),
+            ..
+        } => Some(thread_id.clone()),
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::NewsPost(ShowThreadQueryThreadOnNewsPost { thread_id, .. }),
+            ..
+        } => Some(thread_id.clone()),
+        _ => None,
+    }
+}
+
 // `Model` describes our app state.
 pub struct Model {
     pub query: String,
@@ -565,6 +689,8 @@ pub struct Model {
     pub read_completion_ratio: f64,
     pub content_el: ElRef<HtmlElement>,
     pub versions: Version,
+    pub catchup: Option<Catchup>,
+    pub last_url: Url,
 }
 
 #[derive(Debug)]
@@ -601,6 +727,16 @@ pub enum Context {
     },
 }
 
+pub struct Catchup {
+    pub items: Vec<CatchupItem>,
+}
+
+#[derive(Debug)]
+pub struct CatchupItem {
+    pub id: String,
+    pub seen: bool,
+}
+
 pub struct Tag {
     pub name: String,
     pub bg_color: String,
@@ -614,20 +750,22 @@ pub enum RefreshingState {
     Error(String),
 }
 // `Msg` describes the different events you can modify state with.
+#[derive(strum_macros::Display)]
 pub enum Msg {
     Noop,
     // Tell the client to refresh its state
     Refresh,
     // Tell the client to reload whole page from server
     Reload,
-    // Window has changed size
-    OnResize,
+    // TODO: add GoToUrl
+    OnUrlChanged(subs::UrlChanged),
     // Tell the server to update state
     RefreshStart,
     RefreshDone(Option<gloo_net::Error>),
     NextPage,
     PreviousPage,
     GoToSearchResults,
+
     UpdateQuery(String),
     SearchQuery(String),
 
@@ -651,10 +789,14 @@ pub enum Msg {
     ShowThreadResult(
         Result<graphql_client::Response<graphql::show_thread_query::ResponseData>, gloo_net::Error>,
     ),
+    CatchupRequest {
+        query: String,
+    },
+    CatchupResult(
+        Result<graphql_client::Response<graphql::catchup_query::ResponseData>, gloo_net::Error>,
+    ),
 
-    #[allow(dead_code)]
     SelectionSetNone,
-    #[allow(dead_code)]
     SelectionSetAll,
     SelectionAddTag(String),
     #[allow(dead_code)]
@@ -670,7 +812,14 @@ pub enum Msg {
 
     CopyToClipboard(String),
 
+    ScrollToTop,
     WindowScrolled,
     SetProgress(f64),
     UpdateServerVersion(String),
+
+    CatchupStart,
+    CatchupKeepUnread,
+    CatchupMarkAsRead,
+    CatchupNext,
+    CatchupExit,
 }

web/static/overrides.css

@@ -1,18 +1,18 @@
 html {
-    background-color: black;
+  background-color: black;
 }
 
 .mail-thread a,
 .news-post a {
-    color: var(--color-link) !important;
-    text-decoration: underline;
+  color: var(--color-link) !important;
+  text-decoration: underline;
 }
 
 .mail-thread br,
 .news-post br {
-    display: block;
-    margin-top: 1em;
-    content: " ";
+  display: block;
+  margin-top: 1em;
+  content: " ";
 }
 
 .mail-thread h1,
@@ -23,47 +23,62 @@ html {
 .news-post h2,
 .news-post h3,
 .news-post h4 {
-    margin-top: 1em !important;
-    margin-bottom: 1em !important;
+  margin-top: 1em !important;
+  margin-bottom: 1em !important;
 }
 
 .mail-thread p,
 .news-post p {
-    margin-bottom: 1em;
+  margin-bottom: 1em;
 }
 
 .mail-thread pre,
+.news-post pre {
+  font-family: monospace;
+  background-color: #eee !important;
+  padding: 0.5em;
+  white-space: break-spaces;
+}
+
 .mail-thread code,
-.news-post pre,
 .news-post code {
-    font-family: monospace;
-    background-color: #eee !important;
-    padding: 0.5em !important;
+  font-family: monospace;
+  white-space: break-spaces;
+  background-color: #eee !important;
 }
 
 .mail-thread blockquote {
-    padding-left: 1em;
-    border-left: 2px solid #ddd;
+  padding-left: 1em;
+  border-left: 2px solid #ddd;
 }
 
 .mail-thread ol,
 .mail-thread ul {
-    margin-left: 2em;
+  margin-left: 2em;
+}
+
+.mail-thread .noreply-news-bloomberg-com a {
+  background-color: initial !important;
+}
+
+.mail-thread .noreply-news-bloomberg-com h2 {
+  margin: 0 !important;
+  padding: 0 !important;
 }
 
 /* Hackaday figures have unreadable black on dark grey */
 .news-post figcaption.wp-caption-text {
-    background-color: initial !important;
+  background-color: initial !important;
 }
 
 .news-post.site-nautilus .article-ad,
 .news-post.site-nautilus .primis-ad {
-    display: none !important;
+  display: none !important;
 }
 
 .news-post.site-slashdot .story-byline {
-    display: block !important;
-    height: initial !important;
-    overflow: auto !important;
-    position: static !important;
-}
+  display: block !important;
+  height: initial !important;
+  overflow: auto !important;
+  position: static !important;
+}