chore: Release

Additionally add /test handler that triggers server->client WS message

.gitea/workflows/rust.yml

@@ -1,4 +1,4 @@
-on: [push, pull_request]
+on: [push]
 
 name: Continuous integration
 
@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-            toolchain: stable
+            toolchain: nightly
             target: wasm32-unknown-unknown
       - run: cargo install trunk
       - run: cd web; trunk build
@@ -49,3 +49,19 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
       - run: cargo build
+
+  udeps:
+    name: Disallow unused dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+
+      - name: Run cargo-udeps
+        uses: aig787/cargo-udeps-action@v1
+        with:
+          version: 'latest'
+          args: '--all-targets'
+

Cargo.toml

@@ -1,8 +1,15 @@
 [workspace]
 resolver = "2"
 default-members = ["server"]
-members = ["server", "notmuch", "procmail2notmuch", "shared"]
-#members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+members = ["web", "server", "notmuch", "procmail2notmuch", "shared"]
+
+[workspace.package]
+authors = ["Bill Thiede <git@xinu.tv>"]
+edition = "2021"
+license = "UNLICENSED"
+publish = ["xinu"]
+version = "0.17.28"
+repository = "https://git.z.xinu.tv/wathiede/letterbox"
 
 [profile.dev]
 opt-level = 1

Justfile

@@ -0,0 +1,19 @@
+export CARGO_INCREMENTAL := "0"
+export RUSTFLAGS := "-D warnings"
+
+default:
+    @echo "Run: just patch|minor|major"
+
+major: (_release "major")
+minor: (_release "minor")
+patch: (_release "patch")
+
+sqlx-prepare:
+	cd server; cargo sqlx prepare && git add .sqlx; git commit -m "cargo sqlx prepare" .sqlx || true
+
+pull:
+    git pull
+
+
+_release level: pull sqlx-prepare
+    cargo-release release  -x {{ level }} --workspace --no-confirm --registry=xinu

dev.sh

@@ -3,5 +3,5 @@ tmux new-session -d -s letterbox-dev
 tmux rename-window web
 tmux send-keys "cd web; trunk serve -w ../.git -w ../shared -w ../notmuch -w ./" C-m
 tmux new-window -n server
-tmux send-keys "cd server; cargo watch -c -x run -w ../.git -w ../shared -w ../notmuch -w ./" C-m
+tmux send-keys "cd server; cargo watch -c -w ../.git -w ../shared -w ../notmuch -w ./ -x 'run postgres://newsreader@nixos-07.h.xinu.tv/newsreader ../target/database/newsreader /tmp/letterbox/slurp'" C-m
 tmux attach -d -t letterbox-dev

notmuch/Cargo.toml

@@ -1,22 +1,24 @@
 [package]
 name = "letterbox-notmuch"
-version = "0.3.0"
-edition = "2021"
 exclude = ["/testdata"]
 description = "Wrapper for calling notmuch cli"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 
 [dependencies]
-log = "0.4.14"
+log = "0.4.27"
+mailparse = "0.16.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
-thiserror = "2.0.0"
+thiserror = "2.0.12"
 tracing = "0.1.41"
 
 [dev-dependencies]
 itertools = "0.14.0"
 pretty_assertions = "1"
-rayon = "1.5"
+rayon = "1.10"

notmuch/src/lib.rs

@@ -207,15 +207,15 @@
 //! ```
 
 use std::{
+    collections::HashMap,
     ffi::OsStr,
     io::{self},
     path::{Path, PathBuf},
     process::Command,
 };
 
-use log::{error, info};
 use serde::{Deserialize, Serialize};
-use tracing::instrument;
+use tracing::{error, info, instrument, warn};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -270,6 +270,12 @@ pub struct Headers {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub bcc: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "Delivered-To")]
+    pub delivered_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "X-Original-To")]
+    pub x_original_to: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub reply_to: Option<String>,
     pub date: String,
 }
@@ -459,9 +465,11 @@ pub enum NotmuchError {
     StringUtf8Error(#[from] std::string::FromUtf8Error),
     #[error("failed to parse str as int")]
     ParseIntError(#[from] std::num::ParseIntError),
+    #[error("failed to parse mail: {0}")]
+    MailParseError(#[from] mailparse::MailParseError),
 }
 
-#[derive(Default)]
+#[derive(Clone, Default)]
 pub struct Notmuch {
     config_path: Option<PathBuf>,
 }
@@ -494,15 +502,28 @@ impl Notmuch {
         self.tags_for_query("*")
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_add(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
-        self.run_notmuch(["tag", &format!("+{tag}"), search_term])?;
+        self.tags_add(tag, &[search_term])
+    }
+
+    #[instrument(skip_all, fields(tag=tag,search_term=?search_term))]
+    pub fn tags_add(&self, tag: &str, search_term: &[&str]) -> Result<(), NotmuchError> {
+        let tag = format!("+{tag}");
+        let mut args = vec!["tag", &tag];
+        args.extend(search_term);
+        self.run_notmuch(&args)?;
         Ok(())
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_remove(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
-        self.run_notmuch(["tag", &format!("-{tag}"), search_term])?;
+        self.tags_remove(tag, &[search_term])
+    }
+    #[instrument(skip_all, fields(tag=tag,search_term=?search_term))]
+    pub fn tags_remove(&self, tag: &str, search_term: &[&str]) -> Result<(), NotmuchError> {
+        let tag = format!("-{tag}");
+        let mut args = vec!["tag", &tag];
+        args.extend(search_term);
+        self.run_notmuch(&args)?;
         Ok(())
     }
 
@@ -589,6 +610,11 @@ impl Notmuch {
 
     #[instrument(skip_all, fields(id=id,part=part))]
     pub fn show_original_part(&self, id: &MessageId, part: usize) -> Result<Vec<u8>, NotmuchError> {
+        let id = if id.starts_with("id:") {
+            id
+        } else {
+            &format!("id:{id}")
+        };
         let res = self.run_notmuch(["show", "--part", &part.to_string(), id])?;
         Ok(res)
     }
@@ -605,6 +631,80 @@ impl Notmuch {
         Ok(serde_json::from_slice(&res)?)
     }
 
+    #[instrument(skip_all)]
+    pub fn unread_recipients(&self) -> Result<HashMap<String, usize>, NotmuchError> {
+        let slice = self.run_notmuch([
+            "show",
+            "--include-html=false",
+            "--entire-thread=false",
+            "--body=false",
+            "--format=json",
+            // Arbitrary limit to prevent too much work
+            "--limit=1000",
+            "is:unread",
+        ])?;
+        // Notmuch returns JSON with invalid unicode.  So we lossy convert it to a string here and
+        // use that for parsing in rust.
+        let s = String::from_utf8_lossy(&slice);
+        let mut deserializer = serde_json::Deserializer::from_str(&s);
+        deserializer.disable_recursion_limit();
+        let ts: ThreadSet = serde::de::Deserialize::deserialize(&mut deserializer)?;
+        deserializer.end()?;
+        let mut r = HashMap::new();
+        fn collect_from_thread_node(
+            r: &mut HashMap<String, usize>,
+            tn: &ThreadNode,
+        ) -> Result<(), NotmuchError> {
+            let Some(msg) = &tn.0 else {
+                return Ok(());
+            };
+            let mut addrs = vec![];
+            let hdr = &msg.headers.to;
+            if let Some(to) = hdr {
+                addrs.push(to);
+            } else {
+                let hdr = &msg.headers.x_original_to;
+                if let Some(to) = hdr {
+                    addrs.push(to);
+                } else {
+                    let hdr = &msg.headers.delivered_to;
+                    if let Some(to) = hdr {
+                        addrs.push(to);
+                    };
+                };
+            };
+            let hdr = &msg.headers.cc;
+            if let Some(cc) = hdr {
+                addrs.push(cc);
+            };
+            for recipient in addrs {
+                mailparse::addrparse(&recipient)?
+                    .into_inner()
+                    .iter()
+                    .for_each(|a| {
+                        let mailparse::MailAddr::Single(si) = a else {
+                            return;
+                        };
+                        let addr = &si.addr;
+
+                        if addr == "couchmoney@gmail.com" || addr.ends_with("@xinu.tv") {
+                            *r.entry(addr.to_lowercase()).or_default() += 1;
+                        }
+                    });
+            }
+            Ok(())
+        }
+        for t in ts.0 {
+            for tn in t.0 {
+                collect_from_thread_node(&mut r, &tn)?;
+                for sub_tn in tn.1 {
+                    collect_from_thread_node(&mut r, &sub_tn)?;
+                }
+            }
+        }
+        Ok(r)
+    }
+
     fn run_notmuch<I, S>(&self, args: I) -> Result<Vec<u8>, NotmuchError>
     where
         I: IntoIterator<Item = S>,
@@ -617,6 +717,13 @@ impl Notmuch {
         cmd.args(args);
         info!("{:?}", &cmd);
         let out = cmd.output()?;
+        if !out.stderr.is_empty() {
+            warn!(
+                "{:?}: STDERR:\n{}",
+                &cmd,
+                String::from_utf8_lossy(&out.stderr)
+            );
+        }
         Ok(out.stdout)
     }
 }

procmail2notmuch/Cargo.toml

@@ -1,13 +1,20 @@
 [package]
 name = "letterbox-procmail2notmuch"
-version = "0.3.0"
-edition = "2021"
 description = "Tool for generating notmuch rules from procmail"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.69"
+anyhow = "1.0.98"
+clap = { version = "4.5.37", features = ["derive", "env"] }
+letterbox-notmuch = { version = "0.17.9", registry = "xinu" }
+letterbox-shared = { version = "0.17.9", registry = "xinu" }
+serde = { version = "1.0.219", features = ["derive"] }
+sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio"] }
+tokio = { version = "1.44.2", features = ["rt", "macros", "rt-multi-thread"] }

procmail2notmuch/src/main.rs

@@ -1,210 +1,36 @@
-use std::{convert::Infallible, io::Write, str::FromStr};
+use std::{collections::HashMap, io::Write};
 
-#[derive(Debug, Default)]
-enum MatchType {
-    From,
-    Sender,
-    To,
-    Cc,
-    Subject,
-    List,
-    DeliveredTo,
-    XForwardedTo,
-    ReplyTo,
-    XOriginalTo,
-    XSpam,
-    Body,
-    #[default]
-    Unknown,
-}
-#[derive(Debug, Default)]
-struct Match {
-    match_type: MatchType,
-    needle: String,
+use clap::{Parser, Subcommand};
+use letterbox_shared::{cleanup_match, Match, MatchType, Rule};
+use sqlx::{types::Json, PgPool};
+
+#[derive(Debug, Subcommand)]
+enum Mode {
+    Debug,
+    Notmuchrc,
+    LoadSql {
+        #[arg(short, long)]
+        dsn: String,
+    },
 }
 
-#[derive(Debug, Default)]
-struct Rule {
-    matches: Vec<Match>,
-    tags: Vec<String>,
+/// Simple program to greet a person
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Args {
+    #[arg(short, long, default_value = "/home/wathiede/dotfiles/procmailrc")]
+    input: String,
+
+    #[command(subcommand)]
+    mode: Mode,
 }
 
-fn unescape(s: &str) -> String {
-    s.replace('\\', "")
-}
-
-fn cleanup_match(prefix: &str, s: &str) -> String {
-    unescape(&s[prefix.len()..]).replace(".*", "")
-}
-
-mod matches {
-    pub const TO: &'static str = "TO";
-    pub const CC: &'static str = "Cc";
-    pub const TOCC: &'static str = "(TO|Cc)";
-    pub const FROM: &'static str = "From";
-    pub const SENDER: &'static str = "Sender";
-    pub const SUBJECT: &'static str = "Subject";
-    pub const DELIVERED_TO: &'static str = "Delivered-To";
-    pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
-    pub const REPLY_TO: &'static str = "Reply-To";
-    pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
-    pub const LIST_ID: &'static str = "List-ID";
-    pub const X_SPAM: &'static str = "X-Spam";
-    pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
-}
-
-impl FromStr for Match {
-    type Err = Infallible;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        // Examples:
-        //   "* 1^0 ^TOsonyrewards.com@xinu.tv"
-        //   "* ^TOsonyrewards.com@xinu.tv"
-        let mut it = s.split_whitespace().skip(1);
-        let mut needle = it.next().unwrap();
-        if needle == "1^0" {
-            needle = it.next().unwrap();
-        }
-        let mut needle = vec![needle];
-        needle.extend(it);
-        let needle = needle.join(" ");
-        let first = needle.chars().nth(0).unwrap_or(' ');
-        use matches::*;
-        if first == '^' {
-            let needle = &needle[1..];
-            if needle.starts_with(TO) {
-                return Ok(Match {
-                    match_type: MatchType::To,
-                    needle: cleanup_match(TO, needle),
-                });
-            } else if needle.starts_with(FROM) {
-                return Ok(Match {
-                    match_type: MatchType::From,
-                    needle: cleanup_match(FROM, needle),
-                });
-            } else if needle.starts_with(CC) {
-                return Ok(Match {
-                    match_type: MatchType::Cc,
-                    needle: cleanup_match(CC, needle),
-                });
-            } else if needle.starts_with(TOCC) {
-                return Ok(Match {
-                    match_type: MatchType::To,
-                    needle: cleanup_match(TOCC, needle),
-                });
-            } else if needle.starts_with(SENDER) {
-                return Ok(Match {
-                    match_type: MatchType::Sender,
-                    needle: cleanup_match(SENDER, needle),
-                });
-            } else if needle.starts_with(SUBJECT) {
-                return Ok(Match {
-                    match_type: MatchType::Subject,
-                    needle: cleanup_match(SUBJECT, needle),
-                });
-            } else if needle.starts_with(X_ORIGINAL_TO) {
-                return Ok(Match {
-                    match_type: MatchType::XOriginalTo,
-                    needle: cleanup_match(X_ORIGINAL_TO, needle),
-                });
-            } else if needle.starts_with(LIST_ID) {
-                return Ok(Match {
-                    match_type: MatchType::List,
-                    needle: cleanup_match(LIST_ID, needle),
-                });
-            } else if needle.starts_with(REPLY_TO) {
-                return Ok(Match {
-                    match_type: MatchType::ReplyTo,
-                    needle: cleanup_match(REPLY_TO, needle),
-                });
-            } else if needle.starts_with(X_SPAM_FLAG) {
-                return Ok(Match {
-                    match_type: MatchType::XSpam,
-                    needle: '*'.to_string(),
-                });
-            } else if needle.starts_with(X_SPAM) {
-                return Ok(Match {
-                    match_type: MatchType::XSpam,
-                    needle: '*'.to_string(),
-                });
-            } else if needle.starts_with(DELIVERED_TO) {
-                return Ok(Match {
-                    match_type: MatchType::DeliveredTo,
-                    needle: cleanup_match(DELIVERED_TO, needle),
-                });
-            } else if needle.starts_with(X_FORWARDED_TO) {
-                return Ok(Match {
-                    match_type: MatchType::XForwardedTo,
-                    needle: cleanup_match(X_FORWARDED_TO, needle),
-                });
-            } else {
-                unreachable!("needle: '{needle}'")
-            }
-        } else {
-            return Ok(Match {
-                match_type: MatchType::Body,
-                needle: cleanup_match("", &needle),
-            });
-        }
-    }
-}
-
-fn notmuch_from_rules<W: Write>(mut w: W, rules: &[Rule]) -> anyhow::Result<()> {
-    // TODO(wathiede): if reindexing this many tags is too slow, see if combining rules per tag is
-    // faster.
-    let mut lines = Vec::new();
-    for r in rules {
-        for m in &r.matches {
-            for t in &r.tags {
-                if let MatchType::Unknown = m.match_type {
-                    eprintln!("rule has unknown match {:?}", r);
-                    continue;
-                }
-
-                let rule = match m.match_type {
-                    MatchType::From => "from:",
-                    // TODO(wathiede): something more specific?
-                    MatchType::Sender => "from:",
-                    MatchType::To => "to:",
-                    MatchType::Cc => "to:",
-                    MatchType::Subject => "subject:",
-                    MatchType::List => "List-ID:",
-                    MatchType::Body => "",
-                    // TODO(wathiede): these will probably require adding fields to notmuch
-                    // index. Handle them later.
-                    MatchType::DeliveredTo
-                    | MatchType::XForwardedTo
-                    | MatchType::ReplyTo
-                    | MatchType::XOriginalTo
-                    | MatchType::XSpam => continue,
-                    MatchType::Unknown => unreachable!(),
-                };
-                // Preserve unread status if run with --remove-all
-                lines.push(format!(
-                    r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
-                    t, rule, m.needle
-                ));
-                lines.push(format!(
-                    // TODO(wathiede): this assumes `notmuch new` is configured to add
-                    // `tag:unprocessed` to all new mail.
-                    r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
-                    t, rule, m.needle
-                ));
-            }
-        }
-    }
-    lines.sort();
-    for l in lines {
-        writeln!(w, "{l}")?;
-    }
-    Ok(())
-}
-
-fn main() -> anyhow::Result<()> {
-    let input = "/home/wathiede/dotfiles/procmailrc";
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args = Args::parse();
     let mut rules = Vec::new();
     let mut cur_rule = Rule::default();
-    for l in std::fs::read_to_string(input)?.lines() {
+    for l in std::fs::read_to_string(args.input)?.lines() {
         let l = if let Some(idx) = l.find('#') {
             &l[..idx]
         } else {
@@ -222,6 +48,9 @@ fn main() -> anyhow::Result<()> {
         match first {
             ':' => {
                 // start of rule
+
+                // If carbon-copy flag present, don't stop on match
+                cur_rule.stop_on_match = !l.contains('c');
             }
             '*' => {
                 // add to current rule
@@ -230,26 +59,119 @@ fn main() -> anyhow::Result<()> {
             }
             '.' => {
                 // delivery to folder
-                cur_rule.tags.push(cleanup_match(
+                cur_rule.tag = cleanup_match(
                     "",
                     &l.replace('.', "/")
                         .replace(' ', "")
                         .trim_matches('/')
                         .to_string(),
-                ));
+                );
                 rules.push(cur_rule);
                 cur_rule = Rule::default();
             }
+            '/' => cur_rule = Rule::default(), // Ex. /dev/null
             '|' => cur_rule = Rule::default(), // external command
             '$' => {
                 // TODO(wathiede): tag messages with no other tag as 'inbox'
-                cur_rule.tags.push(cleanup_match("", "inbox"));
+                cur_rule.tag = cleanup_match("", "inbox");
                 rules.push(cur_rule);
                 cur_rule = Rule::default();
             } // variable, should only be $DEFAULT in my config
-            _ => panic!("Unhandled first character '{}' {}", first, l),
+            _ => panic!("Unhandled first character '{}'\nLine: {}", first, l),
         }
     }
-    notmuch_from_rules(std::io::stdout(), &rules)?;
+    match args.mode {
+        Mode::Debug => print_rules(&rules),
+        Mode::Notmuchrc => notmuch_from_rules(std::io::stdout(), &rules)?,
+        Mode::LoadSql { dsn } => load_sql(&dsn, &rules).await?,
+    }
+    Ok(())
+}
+
+fn print_rules(rules: &[Rule]) {
+    let mut tally = HashMap::new();
+    for r in rules {
+        for m in &r.matches {
+            *tally.entry(m.match_type).or_insert(0) += 1;
+        }
+    }
+    let mut sorted: Vec<_> = tally.iter().map(|(k, v)| (v, k)).collect();
+    sorted.sort();
+    sorted.reverse();
+    for (v, k) in sorted {
+        println!("{k:?}: {v}");
+    }
+}
+
+fn notmuch_from_rules<W: Write>(mut w: W, rules: &[Rule]) -> anyhow::Result<()> {
+    // TODO(wathiede): if reindexing this many tags is too slow, see if combining rules per tag is
+    // faster.
+    let mut lines = Vec::new();
+    for r in rules {
+        for m in &r.matches {
+            let t = &r.tag;
+            if let MatchType::Unknown = m.match_type {
+                eprintln!("rule has unknown match {:?}", r);
+                continue;
+            }
+
+            let rule = match m.match_type {
+                MatchType::From => "from:",
+                // TODO(wathiede): something more specific?
+                MatchType::Sender => "from:",
+                MatchType::To => "to:",
+                MatchType::Cc => "to:",
+                MatchType::Subject => "subject:",
+                MatchType::ListId => "List-ID:",
+                MatchType::Body => "",
+                // TODO(wathiede): these will probably require adding fields to notmuch
+                // index. Handle them later.
+                MatchType::DeliveredTo
+                | MatchType::XForwardedTo
+                | MatchType::ReplyTo
+                | MatchType::XOriginalTo
+                | MatchType::XSpam => continue,
+                MatchType::Unknown => unreachable!(),
+            };
+            // Preserve unread status if run with --remove-all
+            lines.push(format!(
+                r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
+                t, rule, m.needle
+            ));
+            lines.push(format!(
+                // TODO(wathiede): this assumes `notmuch new` is configured to add
+                // `tag:unprocessed` to all new mail.
+                r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
+                t, rule, m.needle
+            ));
+        }
+    }
+    lines.sort();
+    for l in lines {
+        writeln!(w, "{l}")?;
+    }
+    Ok(())
+}
+
+async fn load_sql(dsn: &str, rules: &[Rule]) -> anyhow::Result<()> {
+    let pool = PgPool::connect(dsn).await?;
+    println!("clearing email_rule table");
+    sqlx::query!("DELETE FROM email_rule")
+        .execute(&pool)
+        .await?;
+
+    for (order, rule) in rules.iter().enumerate() {
+        println!("inserting {order}: {rule:?}");
+        sqlx::query!(
+            r#"
+        INSERT INTO email_rule (sort_order, rule)
+        VALUES ($1, $2)
+            "#,
+            order as i32,
+            Json(rule) as _
+        )
+        .execute(&pool)
+        .await?;
+    }
     Ok(())
 }

server/.sqlx/query-6c5b0a96f45f78795732ea428cc01b4eab28b7150aa37387e7439a6b0b58e88c.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT rule as \"rule: Json<Rule>\"\n        FROM email_rule\n        ORDER BY sort_order\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "rule: Json<Rule>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6c5b0a96f45f78795732ea428cc01b4eab28b7150aa37387e7439a6b0b58e88c"
+}

server/.sqlx/query-dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nSELECT id\nFROM feed\nWHERE slug = $1\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "dabd12987369cb273c0191d46645c376439d246d5a697340574c6afdac93d2cc"
-}

server/.sqlx/query-e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nINSERT INTO feed ( name, slug, url, homepage, selector )\nVALUES ( $1, $2, $3, '', '' )\nRETURNING id\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "e2a448aaf4fe92fc1deda10bf844f6b9225d35758cba7c9f337c1a730aee41bd"
-}

server/Cargo.toml

@@ -1,57 +1,62 @@
 [package]
 name = "letterbox-server"
-version = "0.3.0"
-edition = "2021"
 default-run = "letterbox-server"
 description = "Backend for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ammonia = "4.0.0"
-anyhow = "1.0.79"
+ammonia = "4.1.0"
+anyhow = "1.0.98"
+askama = { version = "0.14.0", features = ["derive"] }
 async-graphql = { version = "7", features = ["log"] }
-async-graphql-rocket = "7"
-async-trait = "0.1.81"
-build-info = "0.0.39"
-cacher = { version = "0.1.0", registry = "xinu" }
-chrono = "0.4.39"
-clap = { version = "4.5.23", features = ["derive"] }
-css-inline = "0.14.0"
+async-graphql-axum = "7.0.16"
+async-trait = "0.1.88"
+axum = { version = "0.8.3", features = ["ws"] }
+axum-macros = "0.5.0"
+build-info = "0.0.41"
+cacher = { version = "0.2.0", registry = "xinu" }
+chrono = "0.4.40"
+clap = { version = "4.5.37", features = ["derive"] }
+css-inline = "0.17.0"
+flate2 = "1.1.2"
 futures = "0.3.31"
+headers = "0.4.0"
 html-escape = "0.2.13"
+letterbox-notmuch = { path = "../notmuch", version = "0.17.28", registry = "xinu" }
+letterbox-shared = { path = "../shared", version = "0.17.28", registry = "xinu" }
 linkify = "0.10.0"
-log = "0.4.17"
-lol_html = "2.0.0"
-mailparse = "0.16.0"
+lol_html = "2.3.0"
+mailparse = "0.16.1"
 maplit = "1.0.2"
 memmap = "0.7.0"
-opentelemetry = "0.28.0"
+quick-xml = { version = "0.38.1", features = ["serialize"] }
 regex = "1.11.1"
-reqwest = { version = "0.12.7", features = ["blocking"] }
-rocket = { version = "0.5.0-rc.2", features = ["json"] }
-rocket_cors = "0.6.0"
-scraper = "0.22.0"
-serde = { version = "1.0.147", features = ["derive"] }
-serde_json = "1.0.87"
-sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
-tantivy = { version = "0.22.0", optional = true }
-thiserror = "2.0.0"
-tokio = "1.26.0"
+reqwest = { version = "0.12.15", features = ["blocking"] }
+scraper = "0.23.1"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.140"
+sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio", "time"] }
+tantivy = { version = "0.24.1", optional = true }
+thiserror = "2.0.12"
+tokio = "1.44.2"
+tower-http = { version = "0.6.2", features = ["trace"] }
 tracing = "0.1.41"
-url = "2.5.2"
+url = "2.5.4"
 urlencoding = "2.1.3"
-#xtracing = { path = "../../xtracing" }
 #xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
-xtracing = { version = "0.2.0", registry = "xinu" }
-letterbox-notmuch = { version = "0.1.0", registry = "xinu" }
-letterbox-shared = { version = "0.1.0", registry = "xinu" }
+#xtracing = { path = "../../xtracing" }
+xtracing = { version = "0.3.2", registry = "xinu" }
+zip = "4.3.0"
 
 [build-dependencies]
-build-info-build = "0.0.39"
+build-info-build = "0.0.41"
 
 [features]
 #default = [ "tantivy" ]

server/Rocket.toml

@@ -5,7 +5,6 @@ newsreader_database_url = "postgres://newsreader@nixos-07.h.xinu.tv/newsreader"
 newsreader_tantivy_db_path = "../target/database/newsreader"
 
 [debug]
-address = "0.0.0.0"
 port = 9345
 # Uncomment to make it production like.
 #log_level = "critical"

server/build.rs

@@ -2,4 +2,5 @@ fn main() {
     // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
     // and `build_info::format!` in the main program.
     build_info_build::build_script();
+    println!("cargo:rerun-if-changed=templates");
 }

server/migrations/20250330175930_update-nzfinder-link.sql

@@ -0,0 +1,20 @@
+-- Bad examples:
+ -- https://nzbfinder.ws/getnzb/d2c3e5a08abadd985dccc6a574122892030b6a9a.nzb&i=95972&r=b55082d289937c050dedc203c9653850
+--  https://nzbfinder.ws/getnzb?id=45add174-7da4-4445-bf2b-a67dbbfc07fe.nzb&r=b55082d289937c050dedc203c9653850
+--  https://nzbfinder.ws/api/v1/getnzb?id=82486020-c192-4fa0-a7e7-798d7d72e973.nzb&r=b55082d289937c050dedc203c9653850
+UPDATE nzb_posts
+SET link = 
+regexp_replace(
+    regexp_replace(
+        regexp_replace(
+            link,
+            'https://nzbfinder.ws/getnzb/',
+            'https://nzbfinder.ws/api/v1/getnzb?id='
+            ),
+        'https://nzbfinder.ws/getnzb',
+        'https://nzbfinder.ws/api/v1/getnzb'
+        ),
+    '&r=',
+    '&apikey='
+    ) 
+;

server/migrations/20250419202131_email-rules.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF NOT EXISTS email_rule;
+
+-- Add down migration script here

server/migrations/20250419202131_email-rules.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS email_rule (
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
+    sort_order integer NOT NULL,
+    rule jsonb NOT NULL
+);

server/migrations/20250623193718_disable-feed.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE feed DROP COLUMN IF EXISTS disabled;

server/migrations/20250623193718_disable-feed.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE feed ADD disabled boolean;

server/src/bin/email2db.rs

@@ -1,22 +0,0 @@
-use clap::Parser;
-use letterbox_server::mail::read_mail_to_db;
-use sqlx::postgres::PgPool;
-
-/// Add certain emails as posts in newsfeed app.
-#[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-    /// DB URL, something like postgres://newsreader@nixos-07.h.xinu.tv/newsreader
-    #[arg(short, long)]
-    db_url: String,
-    /// path to parse
-    path: String,
-}
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
-    let args = Args::parse();
-    let pool = PgPool::connect(&args.db_url).await?;
-    read_mail_to_db(&pool, &args.path).await?;
-    Ok(())
-}

server/src/bin/letterbox-server.rs

@@ -1,114 +1,101 @@
 // Rocket generates a lot of warnings for handlers
 // TODO: figure out why
 #![allow(unreachable_patterns)]
-#[macro_use]
-extern crate rocket;
-use std::{error::Error, io::Cursor, str::FromStr};
+use std::{error::Error, net::SocketAddr, sync::Arc, time::Duration};
 
-use async_graphql::{extensions, http::GraphiQLSource, EmptySubscription, Schema};
-use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
+use async_graphql::{extensions, http::GraphiQLSource, Schema};
+use async_graphql_axum::{GraphQL, GraphQLSubscription};
+//allows to extract the IP of connecting user
+use axum::extract::connect_info::ConnectInfo;
+use axum::{
+    extract::{self, ws::WebSocketUpgrade, Query, State},
+    http::{header, StatusCode},
+    response::{self, IntoResponse, Response},
+    routing::{any, get, post},
+    Router,
+};
 use cacher::FilesystemCacher;
-use letterbox_notmuch::{Notmuch, NotmuchError, ThreadSet};
+use clap::Parser;
+use letterbox_notmuch::Notmuch;
 #[cfg(feature = "tantivy")]
 use letterbox_server::tantivy::TantivyConnection;
 use letterbox_server::{
-    config::Config,
-    error::ServerError,
-    graphql::{Attachment, GraphqlSchema, Mutation, QueryRoot},
-    nm::{attachment_bytes, cid_attachment_bytes},
+    graphql::{compute_catchup_ids, Attachment, MutationRoot, QueryRoot, SubscriptionRoot},
+    nm::{attachment_bytes, cid_attachment_bytes, label_unprocessed},
+    ws::ConnectionTracker,
 };
-use rocket::{
-    fairing::AdHoc,
-    http::{ContentType, Header},
-    request::Request,
-    response::{content, Debug, Responder},
-    serde::json::Json,
-    Response, State,
-};
-use rocket_cors::{AllowedHeaders, AllowedOrigins};
+use letterbox_shared::WebsocketMessage;
+use serde::Deserialize;
 use sqlx::postgres::PgPool;
+use tokio::{net::TcpListener, sync::Mutex};
+use tower_http::trace::{DefaultMakeSpan, TraceLayer};
+use tracing::{error, info};
 
-#[get("/show/<query>/pretty")]
-async fn show_pretty(
-    nm: &State<Notmuch>,
-    query: &str,
-) -> Result<Json<ThreadSet>, Debug<ServerError>> {
-    let query = urlencoding::decode(query).map_err(|e| ServerError::from(NotmuchError::from(e)))?;
-    let res = nm.show(&query).map_err(ServerError::from)?;
-    Ok(Json(res))
+// Make our own error that wraps `ServerError`.
+struct AppError(letterbox_server::ServerError);
+
+// Tell axum how to convert `AppError` into a response.
+impl IntoResponse for AppError {
+    fn into_response(self) -> Response {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            format!("Something went wrong: {}", self.0),
+        )
+            .into_response()
+    }
 }
-
-#[get("/show/<query>")]
-async fn show(nm: &State<Notmuch>, query: &str) -> Result<Json<ThreadSet>, Debug<NotmuchError>> {
-    let query = urlencoding::decode(query).map_err(NotmuchError::from)?;
-    let res = nm.show(&query)?;
-    Ok(Json(res))
-}
-
-struct InlineAttachmentResponder(Attachment);
-
-impl<'r, 'o: 'r> Responder<'r, 'o> for InlineAttachmentResponder {
-    fn respond_to(self, _: &'r Request<'_>) -> rocket::response::Result<'o> {
-        let mut resp = Response::build();
-        if let Some(filename) = self.0.filename {
-            resp.header(Header::new(
-                "Content-Disposition",
-                format!(r#"inline; filename="{}""#, filename),
-            ));
-        }
-        if let Some(content_type) = self.0.content_type {
-            if let Some(ct) = ContentType::parse_flexible(&content_type) {
-                resp.header(ct);
-            }
-        }
-        resp.sized_body(self.0.bytes.len(), Cursor::new(self.0.bytes))
-            .ok()
+// This enables using `?` on functions that return `Result<_, letterbox_server::Error>` to turn them into
+// `Result<_, AppError>`. That way you don't need to do that manually.
+impl<E> From<E> for AppError
+where
+    E: Into<letterbox_server::ServerError>,
+{
+    fn from(err: E) -> Self {
+        Self(err.into())
     }
 }
 
-struct DownloadAttachmentResponder(Attachment);
-
-impl<'r, 'o: 'r> Responder<'r, 'o> for DownloadAttachmentResponder {
-    fn respond_to(self, _: &'r Request<'_>) -> rocket::response::Result<'o> {
-        let mut resp = Response::build();
-        if let Some(filename) = self.0.filename {
-            resp.header(Header::new(
-                "Content-Disposition",
-                format!(r#"attachment; filename="{}""#, filename),
-            ));
-        }
-        if let Some(content_type) = self.0.content_type {
-            if let Some(ct) = ContentType::parse_flexible(&content_type) {
-                resp.header(ct);
-            }
-        }
-        resp.sized_body(self.0.bytes.len(), Cursor::new(self.0.bytes))
-            .ok()
+fn inline_attachment_response(attachment: Attachment) -> impl IntoResponse {
+    info!("attachment filename {:?}", attachment.filename);
+    let mut hdr_map = headers::HeaderMap::new();
+    if let Some(filename) = attachment.filename {
+        hdr_map.insert(
+            header::CONTENT_DISPOSITION,
+            format!(r#"inline; filename="{}""#, filename)
+                .parse()
+                .unwrap(),
+        );
     }
+    if let Some(ct) = attachment.content_type {
+        hdr_map.insert(header::CONTENT_TYPE, ct.parse().unwrap());
+    }
+    info!("hdr_map {hdr_map:?}");
+    (hdr_map, attachment.bytes).into_response()
 }
 
-#[get("/cid/<id>/<cid>")]
-async fn view_cid(
-    nm: &State<Notmuch>,
-    id: &str,
-    cid: &str,
-) -> Result<InlineAttachmentResponder, Debug<ServerError>> {
-    let mid = if id.starts_with("id:") {
-        id.to_string()
-    } else {
-        format!("id:{}", id)
-    };
-    info!("view cid attachment {mid} {cid}");
-    let attachment = cid_attachment_bytes(nm, &mid, &cid)?;
-    Ok(InlineAttachmentResponder(attachment))
+fn download_attachment_response(attachment: Attachment) -> impl IntoResponse {
+    info!("attachment filename {:?}", attachment.filename);
+    let mut hdr_map = headers::HeaderMap::new();
+    if let Some(filename) = attachment.filename {
+        hdr_map.insert(
+            header::CONTENT_DISPOSITION,
+            format!(r#"attachment; filename="{}""#, filename)
+                .parse()
+                .unwrap(),
+        );
+    }
+    if let Some(ct) = attachment.content_type {
+        hdr_map.insert(header::CONTENT_TYPE, ct.parse().unwrap());
+    }
+    info!("hdr_map {hdr_map:?}");
+    (hdr_map, attachment.bytes).into_response()
 }
 
-#[get("/view/attachment/<id>/<idx>/<_>")]
+#[axum_macros::debug_handler]
 async fn view_attachment(
-    nm: &State<Notmuch>,
-    id: &str,
-    idx: &str,
-) -> Result<InlineAttachmentResponder, Debug<ServerError>> {
+    State(AppState { nm, .. }): State<AppState>,
+    extract::Path((id, idx, _)): extract::Path<(String, String, String)>,
+) -> Result<impl IntoResponse, AppError> {
     let mid = if id.starts_with("id:") {
         id.to_string()
     } else {
@@ -119,16 +106,14 @@ async fn view_attachment(
         .split('.')
         .map(|s| s.parse().expect("not a usize"))
         .collect();
-    let attachment = attachment_bytes(nm, &mid, &idx)?;
-    Ok(InlineAttachmentResponder(attachment))
+    let attachment = attachment_bytes(&nm, &mid, &idx)?;
+    Ok(inline_attachment_response(attachment))
 }
 
-#[get("/download/attachment/<id>/<idx>/<_>")]
 async fn download_attachment(
-    nm: &State<Notmuch>,
-    id: &str,
-    idx: &str,
-) -> Result<DownloadAttachmentResponder, Debug<ServerError>> {
+    State(AppState { nm, .. }): State<AppState>,
+    extract::Path((id, idx, _)): extract::Path<(String, String, String)>,
+) -> Result<impl IntoResponse, AppError> {
     let mid = if id.starts_with("id:") {
         id.to_string()
     } else {
@@ -139,102 +124,220 @@ async fn download_attachment(
         .split('.')
         .map(|s| s.parse().expect("not a usize"))
         .collect();
-    let attachment = attachment_bytes(nm, &mid, &idx)?;
-    Ok(DownloadAttachmentResponder(attachment))
+    let attachment = attachment_bytes(&nm, &mid, &idx)?;
+    Ok(download_attachment_response(attachment))
 }
 
-#[get("/original/<id>")]
-async fn original(
-    nm: &State<Notmuch>,
-    id: &str,
-) -> Result<(ContentType, Vec<u8>), Debug<NotmuchError>> {
+async fn view_cid(
+    State(AppState { nm, .. }): State<AppState>,
+    extract::Path((id, cid)): extract::Path<(String, String)>,
+) -> Result<impl IntoResponse, AppError> {
     let mid = if id.starts_with("id:") {
         id.to_string()
     } else {
         format!("id:{}", id)
     };
-    let res = nm.show_original(&mid)?;
-    Ok((ContentType::Plain, res))
+    info!("view cid attachment {mid} {cid}");
+    let attachment = cid_attachment_bytes(&nm, &mid, &cid)?;
+    Ok(inline_attachment_response(attachment))
 }
 
-#[rocket::get("/")]
-fn graphiql() -> content::RawHtml<String> {
-    content::RawHtml(GraphiQLSource::build().endpoint("/api/graphql").finish())
+// TODO make this work with gitea message ids like `wathiede/letterbox/pulls/91@git.z.xinu.tv`
+async fn view_original(
+    State(AppState { nm, .. }): State<AppState>,
+    extract::Path(id): extract::Path<String>,
+) -> Result<impl IntoResponse, AppError> {
+    info!("view_original {id}");
+    let bytes = nm.show_original(&id)?;
+    let s = String::from_utf8_lossy(&bytes).to_string();
+    Ok(s.into_response())
 }
 
-#[rocket::get("/graphql?<query..>")]
-async fn graphql_query(schema: &State<GraphqlSchema>, query: GraphQLQuery) -> GraphQLResponse {
-    query.execute(schema.inner()).await
+async fn graphiql() -> impl IntoResponse {
+    response::Html(
+        GraphiQLSource::build()
+            .endpoint("/api/graphql/")
+            .subscription_endpoint("/api/graphql/ws")
+            .finish(),
+    )
 }
 
-#[rocket::post("/graphql", data = "<request>", format = "application/json")]
-async fn graphql_request(
-    schema: &State<GraphqlSchema>,
-    request: GraphQLRequest,
-) -> GraphQLResponse {
-    request.execute(schema.inner()).await
+async fn start_ws(
+    ws: WebSocketUpgrade,
+    ConnectInfo(addr): ConnectInfo<SocketAddr>,
+    State(AppState {
+        connection_tracker, ..
+    }): State<AppState>,
+) -> impl IntoResponse {
+    info!("intiating websocket connection for {addr}");
+    ws.on_upgrade(async move |socket| connection_tracker.lock().await.add_peer(socket, addr).await)
 }
 
-#[rocket::main]
+#[derive(Debug, Deserialize)]
+struct NotificationParams {
+    delay_ms: Option<u64>,
+    num_unprocessed: Option<usize>,
+}
+
+async fn send_refresh_websocket_handler(
+    State(AppState {
+        nm,
+        pool,
+        connection_tracker,
+        ..
+    }): State<AppState>,
+    params: Query<NotificationParams>,
+) -> impl IntoResponse {
+    info!("send_refresh_websocket_handler params {params:?}");
+    if let Some(delay_ms) = params.delay_ms {
+        let delay = Duration::from_millis(delay_ms);
+        info!("sleeping {delay:?}");
+        tokio::time::sleep(delay).await;
+    }
+    let limit = match params.num_unprocessed {
+        Some(0) => None,
+        Some(limit) => Some(limit),
+        None => Some(10),
+    };
+
+    let mut ids = None;
+    match label_unprocessed(&nm, &pool, false, limit, "tag:unprocessed").await {
+        Ok(i) => ids = Some(i),
+        Err(err) => error!("Failed to label_unprocessed: {err:?}"),
+    };
+    connection_tracker
+        .lock()
+        .await
+        .send_message_all(WebsocketMessage::RefreshMessages)
+        .await;
+    if let Some(ids) = ids {
+        format!("{ids:?}")
+    } else {
+        "refresh triggered".to_string()
+    }
+}
+
+async fn watch_new(
+    nm: Notmuch,
+    pool: PgPool,
+    conn_tracker: Arc<Mutex<ConnectionTracker>>,
+    poll_time: Duration,
+) -> Result<(), async_graphql::Error> {
+    async fn watch_new_iteration(
+        nm: &Notmuch,
+        pool: &PgPool,
+        conn_tracker: Arc<Mutex<ConnectionTracker>>,
+        old_ids: &[String],
+    ) -> Result<Vec<String>, async_graphql::Error> {
+        let ids = compute_catchup_ids(&nm, &pool, "is:unread").await?;
+        info!("old_ids: {} ids: {}", old_ids.len(), ids.len());
+        if old_ids != ids {
+            label_unprocessed(&nm, &pool, false, Some(100), "tag:unprocessed").await?;
+            conn_tracker
+                .lock()
+                .await
+                .send_message_all(WebsocketMessage::RefreshMessages)
+                .await
+        }
+        Ok(ids)
+    }
+    let mut old_ids = Vec::new();
+    loop {
+        old_ids = match watch_new_iteration(&nm, &pool, conn_tracker.clone(), &old_ids).await {
+            Ok(old_ids) => old_ids,
+            Err(err) => {
+                error!("watch_new_iteration failed: {err:?}");
+                continue;
+            }
+        };
+        tokio::time::sleep(poll_time).await;
+    }
+}
+
+#[derive(Clone)]
+struct AppState {
+    nm: Notmuch,
+    pool: PgPool,
+    connection_tracker: Arc<Mutex<ConnectionTracker>>,
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    #[arg(short, long, default_value = "0.0.0.0:9345")]
+    addr: SocketAddr,
+    newsreader_database_url: String,
+    newsreader_tantivy_db_path: String,
+    slurp_cache_path: String,
+}
+
+#[tokio::main]
 async fn main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
     let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
     build_info::build_info!(fn bi);
     info!("Build Info: {}", letterbox_shared::build_version(bi));
-    let allowed_origins = AllowedOrigins::all();
-    let cors = rocket_cors::CorsOptions {
-        allowed_origins,
-        allowed_methods: vec!["Get"]
-            .into_iter()
-            .map(|s| FromStr::from_str(s).unwrap())
-            .collect(),
-        allowed_headers: AllowedHeaders::some(&["Authorization", "Accept"]),
-        allow_credentials: true,
-        ..Default::default()
+    if !std::fs::exists(&cli.slurp_cache_path)? {
+        info!("Creating slurp cache @ '{}'", &cli.slurp_cache_path);
+        std::fs::create_dir_all(&cli.slurp_cache_path)?;
     }
-    .to_cors()?;
-
-    let rkt = rocket::build()
-        .mount(
-            letterbox_shared::urls::MOUNT_POINT,
-            routes![
-                original,
-                show_pretty,
-                show,
-                graphql_query,
-                graphql_request,
-                graphiql,
-                view_cid,
-                view_attachment,
-                download_attachment,
-            ],
-        )
-        .attach(cors)
-        .attach(AdHoc::config::<Config>());
-
-    let config: Config = rkt.figment().extract()?;
-    if !std::fs::exists(&config.slurp_cache_path)? {
-        info!("Creating slurp cache @ '{}'", &config.slurp_cache_path);
-        std::fs::create_dir_all(&config.slurp_cache_path)?;
-    }
-    let pool = PgPool::connect(&config.newsreader_database_url).await?;
+    let pool = PgPool::connect(&cli.newsreader_database_url).await?;
+    let nm = Notmuch::default();
     sqlx::migrate!("./migrations").run(&pool).await?;
     #[cfg(feature = "tantivy")]
-    let tantivy_conn = TantivyConnection::new(&config.newsreader_tantivy_db_path)?;
+    let tantivy_conn = TantivyConnection::new(&cli.newsreader_tantivy_db_path)?;
 
-    let cacher = FilesystemCacher::new(&config.slurp_cache_path)?;
-    let schema = Schema::build(QueryRoot, Mutation, EmptySubscription)
-        .data(Notmuch::default())
+    let cacher = FilesystemCacher::new(&cli.slurp_cache_path)?;
+    let schema = Schema::build(QueryRoot, MutationRoot, SubscriptionRoot)
+        .data(nm.clone())
         .data(cacher)
         .data(pool.clone());
 
-    #[cfg(feature = "tantivy")]
-    let schema = schema.data(tantivy_conn);
-
     let schema = schema.extension(extensions::Logger).finish();
 
-    let rkt = rkt.manage(schema).manage(pool).manage(Notmuch::default());
-    //.manage(Notmuch::with_config("../notmuch/testdata/notmuch.config"))
+    let connection_tracker = Arc::new(Mutex::new(ConnectionTracker::default()));
+    let ct = Arc::clone(&connection_tracker);
+    let poll_time = Duration::from_secs(60);
+    let _h = tokio::spawn(watch_new(nm.clone(), pool.clone(), ct, poll_time));
 
-    rkt.launch().await?;
+    let api_routes = Router::new()
+        .route(
+            "/download/attachment/{id}/{idx}/{*rest}",
+            get(download_attachment),
+        )
+        .route("/view/attachment/{id}/{idx}/{*rest}", get(view_attachment))
+        .route("/original/{id}", get(view_original))
+        .route("/cid/{id}/{cid}", get(view_cid))
+        .route("/ws", any(start_ws))
+        .route_service("/graphql/ws", GraphQLSubscription::new(schema.clone()))
+        .route(
+            "/graphql/",
+            get(graphiql).post_service(GraphQL::new(schema.clone())),
+        );
+
+    let notification_routes = Router::new()
+        .route("/mail", post(send_refresh_websocket_handler))
+        .route("/news", post(send_refresh_websocket_handler));
+    let app = Router::new()
+        .nest("/api", api_routes)
+        .nest("/notification", notification_routes)
+        .with_state(AppState {
+            nm,
+            pool,
+            connection_tracker,
+        })
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(DefaultMakeSpan::default().include_headers(true)),
+        );
+
+    let listener = TcpListener::bind(cli.addr).await.unwrap();
+    tracing::info!("listening on {}", listener.local_addr().unwrap());
+    axum::serve(
+        listener,
+        app.into_make_service_with_connect_info::<SocketAddr>(),
+    )
+    .await
+    .unwrap();
     Ok(())
 }

server/src/bin/test-labeling.rs

@@ -0,0 +1,39 @@
+use std::error::Error;
+
+use clap::Parser;
+use letterbox_notmuch::Notmuch;
+use letterbox_server::nm::label_unprocessed;
+use sqlx::postgres::PgPool;
+use tracing::info;
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    #[arg(short, long)]
+    newsreader_database_url: String,
+    #[arg(short, long, default_value = "10")]
+    /// Set to 0 to process all matches
+    messages_to_process: usize,
+    #[arg(short, long, default_value = "false")]
+    execute: bool,
+    /// Process messages matching this notmuch query
+    #[arg(short, long, default_value = "tag:unprocessed")]
+    query: String,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
+    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
+    build_info::build_info!(fn bi);
+    info!("Build Info: {}", letterbox_shared::build_version(bi));
+    let pool = PgPool::connect(&cli.newsreader_database_url).await?;
+    let nm = Notmuch::default();
+    let limit = if cli.messages_to_process > 0 {
+        Some(cli.messages_to_process)
+    } else {
+        None
+    };
+    label_unprocessed(&nm, &pool, !cli.execute, limit, &cli.query).await?;
+    Ok(())
+}

server/src/error.rs

@@ -39,4 +39,6 @@ pub enum ServerError {
     QueryParseError(#[from] QueryParserError),
     #[error("impossible: {0}")]
     InfaillibleError(#[from] Infallible),
+    #[error("askama error: {0}")]
+    AskamaError(#[from] askama::Error),
 }

server/src/graphql.rs

@@ -2,20 +2,21 @@ use std::{fmt, str::FromStr};
 
 use async_graphql::{
     connection::{self, Connection, Edge, OpaqueCursor},
-    Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
-    SimpleObject, Union,
+    futures_util::Stream,
+    Context, Enum, Error, FieldResult, InputObject, Object, Schema, SimpleObject, Subscription,
+    Union,
 };
 use cacher::FilesystemCacher;
+use futures::stream;
 use letterbox_notmuch::Notmuch;
-use log::info;
 use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPool;
 use tokio::join;
-use tracing::instrument;
+use tracing::{info, instrument};
 
 #[cfg(feature = "tantivy")]
 use crate::tantivy::TantivyConnection;
-use crate::{newsreader, nm, Query};
+use crate::{newsreader, nm, nm::label_unprocessed, Query};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -95,6 +96,10 @@ pub struct Message {
     pub to: Vec<Email>,
     // All CC headers found in email
     pub cc: Vec<Email>,
+    // X-Original-To header found in email
+    pub x_original_to: Option<Email>,
+    // Delivered-To header found in email
+    pub delivered_to: Option<Email>,
     // First Subject header found in email
     pub subject: Option<String>,
     // Parsed Date header, if found and valid
@@ -285,8 +290,7 @@ impl QueryRoot {
         build_info::build_info!(fn bi);
         Ok(letterbox_shared::build_version(bi))
     }
-    #[instrument(skip_all, fields(query=query))]
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn count<'ctx>(&self, ctx: &Context<'ctx>, query: String) -> Result<usize, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -306,10 +310,20 @@ impl QueryRoot {
         info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count} tantivy count {tantivy_count} total {total}");
         Ok(total)
     }
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
+    async fn catchup<'ctx>(
+        &self,
+        ctx: &Context<'ctx>,
+        query: String,
+    ) -> Result<Vec<String>, Error> {
+        let nm = ctx.data_unchecked::<Notmuch>();
+        let pool = ctx.data_unchecked::<PgPool>();
+        compute_catchup_ids(nm, pool, &query).await
+    }
 
     // TODO: this function doesn't get parallelism, possibly because notmuch is sync and blocks,
     // rewrite that with tokio::process:Command
-    #[instrument(skip_all, fields(query=query, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, rid=request_id()))]
     async fn search<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -467,7 +481,7 @@ impl QueryRoot {
         .await?)
     }
 
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
@@ -476,7 +490,7 @@ impl QueryRoot {
         tags.append(&mut nm::tags(nm, needs_unread)?);
         Ok(tags)
     }
-    #[instrument(skip_all, fields(thread_id=thread_id, request_id=request_id()))]
+    #[instrument(skip_all, fields(thread_id=thread_id, rid=request_id()))]
     async fn thread<'ctx>(&self, ctx: &Context<'ctx>, thread_id: String) -> Result<Thread, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let cacher = ctx.data_unchecked::<FilesystemCacher>();
@@ -550,10 +564,10 @@ async fn tantivy_search(
         .collect())
 }
 
-pub struct Mutation;
+pub struct MutationRoot;
 #[Object]
-impl Mutation {
-    #[instrument(skip_all, fields(query=query, unread=unread, request_id=request_id()))]
+impl MutationRoot {
+    #[instrument(skip_all, fields(query=query, unread=unread, rid=request_id()))]
     async fn set_read_status<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -572,7 +586,7 @@ impl Mutation {
         nm::set_read_status(nm, &query, unread).await?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_add<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -584,7 +598,7 @@ impl Mutation {
         nm.tag_add(&tag, &query)?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag, request_id=request_id()))]
+    #[instrument(skip_all, fields(query=query, tag=tag, rid=request_id()))]
     async fn tag_remove<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -607,13 +621,17 @@ impl Mutation {
 
         Ok(true)
     }
-    #[instrument(skip_all, fields(request_id=request_id()))]
+    #[instrument(skip_all, fields(rid=request_id()))]
     async fn refresh<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
         let cacher = ctx.data_unchecked::<FilesystemCacher>();
         let pool = ctx.data_unchecked::<PgPool>();
         info!("{}", String::from_utf8_lossy(&nm.new()?));
         newsreader::refresh(pool, cacher).await?;
+
+        // Process email labels
+        label_unprocessed(&nm, &pool, false, Some(10), "tag:unprocessed").await?;
+
         #[cfg(feature = "tantivy")]
         {
             let tantivy = ctx.data_unchecked::<TantivyConnection>();
@@ -624,4 +642,51 @@ impl Mutation {
     }
 }
 
-pub type GraphqlSchema = Schema<QueryRoot, Mutation, EmptySubscription>;
+pub struct SubscriptionRoot;
+#[Subscription]
+impl SubscriptionRoot {
+    async fn values(&self, _ctx: &Context<'_>) -> Result<impl Stream<Item = usize>, Error> {
+        Ok(stream::iter(0..10))
+    }
+}
+
+pub type GraphqlSchema = Schema<QueryRoot, MutationRoot, SubscriptionRoot>;
+
+#[instrument(skip_all, fields(query=query))]
+pub async fn compute_catchup_ids(
+    nm: &Notmuch,
+    pool: &PgPool,
+    query: &str,
+) -> Result<Vec<String>, Error> {
+    let query: Query = query.parse()?;
+    // TODO: implement optimized versions of fetching just IDs
+    let newsreader_fut = newsreader_search(pool, None, None, None, None, &query);
+    let notmuch_fut = notmuch_search(nm, None, None, None, None, &query);
+    let (newsreader_results, notmuch_results) = join!(newsreader_fut, notmuch_fut);
+
+    let newsreader_results = newsreader_results?;
+    let notmuch_results = notmuch_results?;
+    info!(
+        "newsreader_results ({}) notmuch_results ({})",
+        newsreader_results.len(),
+        notmuch_results.len(),
+    );
+
+    let mut results: Vec<_> = newsreader_results
+        .into_iter()
+        .chain(notmuch_results)
+        .collect();
+    // The leading '-' is to reverse sort
+    results.sort_by_key(|item| match item {
+        ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
+        ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
+    });
+    let ids = results
+        .into_iter()
+        .map(|r| match r {
+            ThreadSummaryCursor::Newsreader(_, ts) => ts.thread,
+            ThreadSummaryCursor::Notmuch(_, ts) => ts.thread,
+        })
+        .collect();
+    Ok(ids)
+}

server/src/lib.rs

@@ -1,9 +1,10 @@
 pub mod config;
 pub mod error;
 pub mod graphql;
-pub mod mail;
 pub mod newsreader;
 pub mod nm;
+pub mod ws;
+
 #[cfg(feature = "tantivy")]
 pub mod tantivy;
 
@@ -18,8 +19,8 @@ use std::{
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
 use css_inline::{CSSInliner, InlineError, InlineOptions};
+pub use error::ServerError;
 use linkify::{LinkFinder, LinkKind};
-use log::{debug, error, info, warn};
 use lol_html::{
     element, errors::RewritingError, html_content::ContentType, rewrite_str, text,
     RewriteStrSettings,
@@ -30,10 +31,10 @@ use reqwest::StatusCode;
 use scraper::{Html, Selector};
 use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
+use tracing::{debug, error, info, warn};
 use url::Url;
 
 use crate::{
-    error::ServerError,
     graphql::{Corpus, ThreadSummary},
     newsreader::is_newsreader_thread,
     nm::is_notmuch_thread_or_id,
@@ -318,13 +319,16 @@ impl<'c> Transformer for SlurpContents<'c> {
         } else {
             let resp = reqwest::get(link.as_str()).await?;
             let status = resp.status();
-            if status.is_server_error() || retryable_status.contains(&status) {
-                return Err(TransformError::RetryableHttpStatusError(
-                    status,
-                    link.to_string(),
-                ));
+            if status.is_server_error() {
+                error!("status error for {link}: {status}");
+                return Ok(html.to_string());
+            }
+            if retryable_status.contains(&status) {
+                error!("retryable error for {link}: {status}");
+                return Ok(html.to_string());
             }
             if !status.is_success() {
+                error!("unsuccessful for {link}: {status}");
                 return Ok(html.to_string());
             }
             let body = resp.text().await?;
@@ -438,6 +442,34 @@ pub fn sanitize_html(
         }
     };
     let mut element_content_handlers = vec![
+        // Remove width and height attributes on elements
+        element!("[width],[height]", |el| {
+            el.remove_attribute("width");
+            el.remove_attribute("height");
+            Ok(())
+        }),
+        // Remove width and height values from inline styles
+        element!("[style]", |el| {
+            let style = el.get_attribute("style").unwrap();
+            let style = style
+                .split(";")
+                .filter(|s| {
+                    let Some((k, _)) = s.split_once(':') else {
+                        return true;
+                    };
+                    match k {
+                        "width" | "max-width" | "min-width" | "height" | "max-height"
+                        | "min-height" => false,
+                        _ => true,
+                    }
+                })
+                .collect::<Vec<_>>()
+                .join(";");
+            if let Err(e) = el.set_attribute("style", &style) {
+                error!("Failed to set style attribute: {e}");
+            }
+            Ok(())
+        }),
         // Open links in new tab
         element!("a[href]", |el| {
             el.set_attribute("target", "_blank").unwrap();
@@ -682,7 +714,7 @@ fn compute_offset_limit(
     first: Option<i32>,
     last: Option<i32>,
 ) -> (i32, i32) {
-    let default_page_size = 100;
+    let default_page_size = 10000;
     match (after, before, first, last) {
         // Reasonable defaults
         (None, None, None, None) => (0, default_page_size),
@@ -773,7 +805,19 @@ impl Query {
         for uid in &self.uids {
             parts.push(uid.clone());
         }
-        parts.extend(self.remainder.clone());
+        for r in &self.remainder {
+            // Rewrite "to:" to include ExtraTo:. ExtraTo: is configured in
+            // notmuch-config to index Delivered-To and X-Original-To headers.
+            if r.starts_with("to:") {
+                parts.push("(".to_string());
+                parts.push(r.to_string());
+                parts.push("OR".to_string());
+                parts.push(r.replace("to:", "ExtraTo:"));
+                parts.push(")".to_string());
+            } else {
+                parts.push(r.to_string());
+            }
+        }
         parts.join(" ")
     }
 }
@@ -793,7 +837,17 @@ impl FromStr for Query {
             if word == "is:unread" {
                 unread_only = true
             } else if word.starts_with("tag:") {
-                tags.push(word["tag:".len()..].to_string());
+                let t = &word["tag:".len()..];
+                // Per-address emails are faked as `tag:@<domain>/<username>`, rewrite to `to:` form
+                if t.starts_with('@') && t.contains('.') {
+                    let t = match t.split_once('/') {
+                        None => format!("to:{t}"),
+                        Some((domain, user)) => format!("to:{user}{domain}"),
+                    };
+                    remainder.push(t);
+                } else {
+                    tags.push(t.to_string());
+                };
 
                 /*
                             } else if word.starts_with("tag:") {
@@ -888,3 +942,21 @@ async fn clean_title(title: &str) -> Result<String, ServerError> {
     }
     Ok(title)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{SanitizeHtml, Transformer};
+
+    #[tokio::test]
+    async fn strip_sizes() -> Result<(), Box<dyn std::error::Error>> {
+        let ss = SanitizeHtml {
+            cid_prefix: "",
+            base_url: &None,
+        };
+        let input = r#"<p width=16 height=16 style="color:blue;width:16px;height:16px;">This el has width and height attributes and inline styles</p>"#;
+        let want = r#"<p style="color:blue;">This el has width and height attributes and inline styles</p>"#;
+        let got = ss.transform(&None, input).await?;
+        assert_eq!(got, want);
+        Ok(())
+    }
+}

server/src/mail.rs

@@ -1,113 +0,0 @@
-use std::{fs::File, io::Read};
-
-use mailparse::{
-    addrparse_header, dateparse, parse_mail, MailHeaderMap, MailParseError, ParsedMail,
-};
-use sqlx::postgres::PgPool;
-use thiserror::Error;
-use tracing::info;
-
-#[derive(Error, Debug)]
-pub enum MailError {
-    #[error("missing from header")]
-    MissingFrom,
-    #[error("missing from header display name")]
-    MissingFromDisplayName,
-    #[error("missing subject header")]
-    MissingSubject,
-    #[error("missing html part")]
-    MissingHtmlPart,
-    #[error("missing message ID")]
-    MissingMessageId,
-    #[error("missing date")]
-    MissingDate,
-    #[error("DB error {0}")]
-    SqlxError(#[from] sqlx::Error),
-    #[error("IO error {0}")]
-    IOError(#[from] std::io::Error),
-    #[error("mail parse error {0}")]
-    MailParseError(#[from] MailParseError),
-}
-
-pub async fn read_mail_to_db(pool: &PgPool, path: &str) -> Result<(), MailError> {
-    let mut file = File::open(path)?;
-    let mut buffer = Vec::new();
-    file.read_to_end(&mut buffer)?;
-    let m = parse_mail(&buffer)?;
-
-    let subject = m
-        .headers
-        .get_first_value("subject")
-        .ok_or(MailError::MissingSubject)?;
-
-    let from = addrparse_header(
-        m.headers
-            .get_first_header("from")
-            .ok_or(MailError::MissingFrom)?,
-    )?;
-    let from = from.extract_single_info().ok_or(MailError::MissingFrom)?;
-    let name = from.display_name.ok_or(MailError::MissingFromDisplayName)?;
-    let slug = name.to_lowercase().replace(' ', "-");
-    let url = from.addr;
-    let message_id = m
-        .headers
-        .get_first_value("Message-ID")
-        .ok_or(MailError::MissingMessageId)?;
-    let uid = &message_id;
-    let feed_id = find_feed(&pool, &name, &slug, &url).await?;
-    let date = dateparse(
-        &m.headers
-            .get_first_value("Date")
-            .ok_or(MailError::MissingDate)?,
-    )?;
-
-    println!("Feed: {feed_id} Subject: {}", subject);
-
-    if let Some(_m) = first_html(&m) {
-        info!("add email {slug} {subject} {message_id} {date} {uid} {url}");
-    } else {
-        return Err(MailError::MissingHtmlPart.into());
-    }
-
-    Ok(())
-}
-fn first_html<'m>(m: &'m ParsedMail<'m>) -> Option<&'m ParsedMail<'m>> {
-    for ele in m.parts() {
-        if ele.ctype.mimetype == "text/html" {
-            return Some(ele);
-        }
-    }
-    None
-}
-async fn find_feed(pool: &PgPool, name: &str, slug: &str, url: &str) -> Result<i32, MailError> {
-    match sqlx::query!(
-        r#"
-SELECT id
-FROM feed
-WHERE slug = $1
-        "#,
-        slug
-    )
-    .fetch_one(pool)
-    .await
-    {
-        Err(sqlx::Error::RowNotFound) => {
-            let rec = sqlx::query!(
-                r#"
-INSERT INTO feed ( name, slug, url, homepage, selector )
-VALUES ( $1, $2, $3, '', '' )
-RETURNING id
-        "#,
-                name,
-                slug,
-                url
-            )
-            .fetch_one(pool)
-            .await?;
-
-            return Ok(rec.id);
-        }
-        Ok(rec) => return Ok(rec.id),
-        Err(e) => return Err(e.into()),
-    };
-}

server/src/newsreader.rs

@@ -3,11 +3,10 @@ use std::collections::HashMap;
 use cacher::FilesystemCacher;
 use futures::{stream::FuturesUnordered, StreamExt};
 use letterbox_shared::compute_color;
-use log::{error, info};
 use maplit::hashmap;
 use scraper::Selector;
 use sqlx::postgres::PgPool;
-use tracing::instrument;
+use tracing::{error, info, instrument};
 use url::Url;
 
 use crate::{
@@ -353,6 +352,9 @@ fn slurp_contents_selectors() -> HashMap<String, Vec<Selector>> {
         "natwelch.com".to_string() => vec![
             Selector::parse("article div.prose").unwrap(),
         ],
+        "seiya.me".to_string() => vec![
+            Selector::parse("header + div").unwrap(),
+        ],
         "rustacean-station.org".to_string() => vec![
             Selector::parse("article").unwrap(),
         ],

server/src/nm.rs

@@ -1,16 +1,19 @@
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     fs::File,
-    hash::{DefaultHasher, Hash, Hasher},
-    time::Instant,
+    io::Cursor,
 };
 
+use askama::Template;
+use chrono::{TimeZone, Utc};
 use letterbox_notmuch::Notmuch;
-use log::{error, info, warn};
+use letterbox_shared::{compute_color, Rule};
 use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
 use memmap::MmapOptions;
-use sqlx::PgPool;
-use tracing::instrument;
+use quick_xml::de::from_str as xml_from_str;
+use sqlx::{types::Json, PgPool};
+use tracing::{error, info, info_span, instrument, warn};
+use zip::ZipArchive;
 
 use crate::{
     compute_offset_limit,
@@ -22,6 +25,7 @@ use crate::{
     linkify_html, InlineStyle, Query, SanitizeHtml, Transformer,
 };
 
+const APPLICATION_ZIP: &'static str = "application/zip";
 const IMAGE_JPEG: &'static str = "image/jpeg";
 const IMAGE_PJPEG: &'static str = "image/pjpeg";
 const IMAGE_PNG: &'static str = "image/png";
@@ -107,7 +111,6 @@ pub async fn search(
 
 #[instrument(name="nm::tags", skip_all, fields(needs_unread=needs_unread))]
 pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
-    let now = Instant::now();
     let unread_msg_cnt: HashMap<String, usize> = if needs_unread {
         // 10000 is an arbitrary number, if there's more than 10k unread messages, we'll
         //  get an inaccurate count.
@@ -123,13 +126,11 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
     } else {
         HashMap::new()
     };
-    let tags = nm
+    let tags: Vec<_> = nm
         .tags()?
         .into_iter()
         .map(|tag| {
-            let mut hasher = DefaultHasher::new();
-            tag.hash(&mut hasher);
-            let hex = format!("#{:06x}", hasher.finish() % (1 << 24));
+            let hex = compute_color(&tag);
             let unread = if needs_unread {
                 *unread_msg_cnt.get(&tag).unwrap_or(&0)
             } else {
@@ -142,8 +143,24 @@ pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
                 unread,
             }
         })
+        .chain(
+            nm.unread_recipients()?
+                .into_iter()
+                .filter_map(|(name, unread)| {
+                    let Some(idx) = name.find('@') else {
+                        return None;
+                    };
+                    let name = format!("{}/{}", &name[idx..], &name[..idx]);
+                    let bg_color = compute_color(&name);
+                    Some(Tag {
+                        name,
+                        fg_color: "white".to_string(),
+                        bg_color,
+                        unread,
+                    })
+                }),
+        )
         .collect();
-    info!("Fetching tags took {} seconds", now.elapsed().as_secs_f32());
     Ok(tags)
 }
 
@@ -187,6 +204,8 @@ pub async fn thread(
 
         let to = email_addresses(&path, &m, "to")?;
         let cc = email_addresses(&path, &m, "cc")?;
+        let delivered_to = email_addresses(&path, &m, "delivered-to")?.pop();
+        let x_original_to = email_addresses(&path, &m, "x-original-to")?.pop();
         let subject = m.headers.get_first_value("subject");
         let timestamp = m
             .headers
@@ -306,6 +325,8 @@ pub async fn thread(
             body,
             path,
             attachments,
+            delivered_to,
+            x_original_to,
         });
     }
     messages.reverse();
@@ -325,7 +346,7 @@ pub async fn thread(
 }
 
 fn email_addresses(
-    path: &str,
+    _path: &str,
     m: &ParsedMail,
     header_name: &str,
 ) -> Result<Vec<Email>, ServerError> {
@@ -336,9 +357,7 @@ fn email_addresses(
                 for ma in mal.into_inner() {
                     match ma {
                         mailparse::MailAddr::Group(gi) => {
-                            if !gi.group_name.contains("ndisclosed") {
-                                println!("[{path}][{header_name}] Group: {gi}");
-                            }
+                            if !gi.group_name.contains("ndisclosed") {}
                         }
                         mailparse::MailAddr::Single(s) => addrs.push(Email {
                             name: s.display_name,
@@ -434,6 +453,7 @@ fn extract_body(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Ser
         MULTIPART_MIXED => extract_mixed(m, part_addr),
         MULTIPART_ALTERNATIVE => extract_alternative(m, part_addr),
         MULTIPART_RELATED => extract_related(m, part_addr),
+        APPLICATION_ZIP => extract_zip(m),
         _ => extract_unhandled(m),
     };
     if let Err(err) = ret {
@@ -443,6 +463,71 @@ fn extract_body(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Ser
     ret
 }
 
+const APPLICATION_GZIP: &'static str = "application/gzip";
+
+fn extract_zip(m: &ParsedMail) -> Result<Body, ServerError> {
+    if let Ok(zip_bytes) = m.get_body_raw() {
+        if let Ok(mut archive) = ZipArchive::new(Cursor::new(&zip_bytes)) {
+            for i in 0..archive.len() {
+                if let Ok(mut file) = archive.by_index(i) {
+                    let name = file.name().to_lowercase();
+                    // Google DMARC reports are typically named like "google.com!example.com!...xml"
+                    // and may or may not contain "dmarc" in the filename.
+                    if name.ends_with(".xml")
+                        && (name.contains("dmarc") || name.starts_with("google.com!"))
+                    {
+                        let mut xml = String::new();
+                        use std::io::Read;
+                        if file.read_to_string(&mut xml).is_ok() {
+                            match parse_dmarc_report(&xml) {
+                                Ok(report) => {
+                                    return Ok(Body::html(format!(
+                                        "<div class=\"dmarc-report\">Google DMARC report summary:<br>{}</div>",
+                                        report
+                                    )));
+                                }
+                                Err(e) => {
+                                    return Ok(Body::html(format!(
+                                        "<div class=\"dmarc-report-error\">Failed to parse DMARC report XML: {}</div>",
+                                        e
+                                    )));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // If no DMARC report found, fall through to unhandled
+    extract_unhandled(m)
+}
+
+fn extract_gzip(m: &ParsedMail) -> Result<Body, ServerError> {
+    if let Ok(gz_bytes) = m.get_body_raw() {
+        let mut decoder = flate2::read::GzDecoder::new(&gz_bytes[..]);
+        let mut xml = String::new();
+        use std::io::Read;
+        if decoder.read_to_string(&mut xml).is_ok() {
+            match parse_dmarc_report(&xml) {
+                Ok(report) => {
+                    return Ok(Body::html(format!(
+                        "<div class=\"dmarc-report\">Microsoft DMARC report summary:<br>{}</div>",
+                        report
+                    )));
+                }
+                Err(e) => {
+                    return Ok(Body::html(format!(
+                        "<div class=\"dmarc-report-error\">Failed to parse DMARC report XML: {}</div>",
+                        e
+                    )));
+                }
+            }
+        }
+    }
+    extract_unhandled(m)
+}
+
 fn extract_unhandled(m: &ParsedMail) -> Result<Body, ServerError> {
     let msg = format!(
         "Unhandled body content type:\n{}\n{}",
@@ -512,6 +597,7 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
         MULTIPART_RELATED,
         TEXT_HTML,
         TEXT_PLAIN,
+        APPLICATION_GZIP,
     ];
     let mut unhandled_types: Vec<_> = m
         .subparts
@@ -555,6 +641,7 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
                     )));
                 }
             }
+            APPLICATION_GZIP => parts.push(extract_gzip(sp)?),
             mt => parts.push(unhandled_html(MULTIPART_MIXED, mt)),
         }
         part_addr.pop();
@@ -914,3 +1001,443 @@ WHERE
     .await?;
     Ok(row.map(|r| r.url))
 }
+
+/*
+* grab email_rules table from sql
+* For each message with `unprocessed` label
+*   parse the message
+*   pass headers for each message through a matcher using email rules
+*   for each match, add label to message
+*   if any matches were found, remove unprocessed
+* TODO: how to handle inbox label
+*/
+
+#[instrument(name="nm::label_unprocessed", skip_all, fields(dryrun=dryrun, limit=?limit, query=%query))]
+pub async fn label_unprocessed(
+    nm: &Notmuch,
+    pool: &PgPool,
+    dryrun: bool,
+    limit: Option<usize>,
+    query: &str,
+) -> Result<Box<[String]>, ServerError> {
+    use futures::StreamExt;
+    let ids = nm.message_ids(query)?;
+    info!(
+        "Processing {limit:?} of {} messages with '{query}'",
+        ids.len()
+    );
+    let rules: Vec<_> = sqlx::query!(
+        r#"
+        SELECT rule as "rule: Json<Rule>"
+        FROM email_rule
+        ORDER BY sort_order
+        "#,
+    )
+    .fetch(pool)
+    .map(|r| r.unwrap().rule.0)
+    .collect()
+    .await;
+    /*
+        use letterbox_shared::{Match, MatchType};
+        let rules = vec![Rule {
+            stop_on_match: false,
+            matches: vec![Match {
+                match_type: MatchType::From,
+                needle: "eftours".to_string(),
+            }],
+            tag: "EFTours".to_string(),
+        }];
+    */
+    info!("Loaded {} rules", rules.len());
+
+    let limit = limit.unwrap_or(ids.len());
+    let limit = limit.min(ids.len());
+    let ids = &ids[..limit];
+
+    let mut add_mutations = HashMap::new();
+    let mut rm_mutations = HashMap::new();
+    for id in ids {
+        let id = format!("id:{id}");
+        let files = nm.files(&id)?;
+        // Only process the first file path is multiple files have the same id
+        let Some(path) = files.iter().next() else {
+            error!("No files for message-ID {id}");
+            let t = "Letterbox/Bad";
+            nm.tag_add(t, &id)?;
+            let t = "unprocessed";
+            nm.tag_remove(t, &id)?;
+            continue;
+        };
+        let file = File::open(&path)?;
+        info!("parsing {path}");
+        let mmap = unsafe { MmapOptions::new().map(&file)? };
+        let m = match info_span!("parse_mail", path = path).in_scope(|| parse_mail(&mmap)) {
+            Ok(m) => m,
+            Err(err) => {
+                error!("Failed to parse {path}: {err}");
+                let t = "Letterbox/Bad";
+                nm.tag_add(t, &id)?;
+                let t = "unprocessed";
+                nm.tag_remove(t, &id)?;
+                continue;
+            }
+        };
+        let (matched_rule, add_tags) = find_tags(&rules, &m.headers);
+        if matched_rule {
+            if dryrun {
+                info!(
+                    "\nAdd tags: {add_tags:?}\nTo: {} From: {} Subject: {}\n",
+                    m.headers.get_first_value("to").expect("no from header"),
+                    m.headers.get_first_value("from").expect("no from header"),
+                    m.headers
+                        .get_first_value("subject")
+                        .expect("no subject header")
+                );
+            }
+            for t in &add_tags {
+                //nm.tag_add(t, &id)?;
+                add_mutations
+                    .entry(t.to_string())
+                    .or_insert_with(|| Vec::new())
+                    .push(id.clone());
+            }
+            if add_tags.contains("spam") || add_tags.contains("Spam") {
+                //nm.tag_remove("unread", &id)?;
+                let t = "unread".to_string();
+                rm_mutations
+                    .entry(t)
+                    .or_insert_with(|| Vec::new())
+                    .push(id.clone());
+            }
+            if !add_tags.contains("inbox") {
+                //nm.tag_remove("inbox", &id)?;
+                let t = "inbox".to_string();
+                rm_mutations
+                    .entry(t)
+                    .or_insert_with(|| Vec::new())
+                    .push(id.clone());
+            }
+            //nm.tag_remove("unprocessed", &id)?;
+        } else {
+            if add_tags.is_empty() {
+                let t = "Grey".to_string();
+                add_mutations
+                    .entry(t)
+                    .or_insert_with(|| Vec::new())
+                    .push(id.clone());
+            }
+            //nm.tag_remove("inbox", &id)?;
+            let t = "inbox".to_string();
+            rm_mutations
+                .entry(t)
+                .or_insert_with(|| Vec::new())
+                .push(id.clone());
+        }
+        let t = "unprocessed".to_string();
+        rm_mutations
+            .entry(t)
+            .or_insert_with(|| Vec::new())
+            .push(id.clone());
+    }
+    info!("Adding {} distinct labels", add_mutations.len());
+    for (tag, ids) in add_mutations.iter() {
+        info!("  {tag}: {}", ids.len());
+        if !dryrun {
+            let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect();
+            info_span!("tags_add", tag = tag, count = ids.len())
+                .in_scope(|| nm.tags_add(tag, &ids))?;
+        }
+    }
+    info!("Removing {} distinct labels", rm_mutations.len());
+    for (tag, ids) in rm_mutations.iter() {
+        info!("  {tag}: {}", ids.len());
+        if !dryrun {
+            let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect();
+            info_span!("tags_remove", tag = tag, count = ids.len())
+                .in_scope(|| nm.tags_remove(tag, &ids))?;
+        }
+    }
+
+    Ok(ids.into())
+}
+fn find_tags<'a, 'b>(rules: &'a [Rule], headers: &'b [MailHeader]) -> (bool, HashSet<&'a str>) {
+    let mut matched_rule = false;
+    let mut add_tags = HashSet::new();
+    for rule in rules {
+        for hdr in headers {
+            if rule.is_match(&hdr.get_key(), &hdr.get_value()) {
+                //info!("Matched {rule:?}");
+                matched_rule = true;
+                add_tags.insert(rule.tag.as_str());
+                if rule.stop_on_match {
+                    return (true, add_tags);
+                }
+            }
+        }
+    }
+    return (matched_rule, add_tags);
+}
+
+// Add this helper function to parse the DMARC XML and summarize it.
+#[derive(Debug, serde::Deserialize)]
+pub struct FormattedDateRange {
+    pub begin: String,
+    pub end: String,
+}
+
+pub struct FormattedReportMetadata {
+    pub org_name: String,
+    pub email: String,
+    pub report_id: String,
+    pub date_range: Option<FormattedDateRange>,
+}
+
+pub struct FormattedRecord {
+    pub source_ip: String,
+    pub count: String,
+    pub header_from: String,
+    pub disposition: String,
+    pub dkim: String,
+    pub spf: String,
+    pub auth_results: Option<FormattedAuthResults>,
+}
+
+pub struct FormattedAuthResults {
+    pub dkim: Vec<FormattedAuthDKIM>,
+    pub spf: Vec<FormattedAuthSPF>,
+}
+
+pub struct FormattedAuthDKIM {
+    pub domain: String,
+    pub result: String,
+    pub selector: String,
+}
+
+pub struct FormattedAuthSPF {
+    pub domain: String,
+    pub result: String,
+    pub scope: String,
+}
+
+pub struct FormattedPolicyPublished {
+    pub domain: String,
+    pub adkim: String,
+    pub aspf: String,
+    pub p: String,
+    pub sp: String,
+    pub pct: String,
+}
+
+pub struct FormattedFeedback {
+    pub report_metadata: Option<FormattedReportMetadata>,
+    pub policy_published: Option<FormattedPolicyPublished>,
+    pub record: Option<Vec<FormattedRecord>>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct Feedback {
+    pub report_metadata: Option<ReportMetadata>,
+    pub policy_published: Option<PolicyPublished>,
+    pub record: Option<Vec<Record>>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct ReportMetadata {
+    pub org_name: Option<String>,
+    pub email: Option<String>,
+    pub report_id: Option<String>,
+    pub date_range: Option<DateRange>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct DateRange {
+    pub begin: Option<u64>,
+    pub end: Option<u64>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct PolicyPublished {
+    pub domain: Option<String>,
+    pub adkim: Option<String>,
+    pub aspf: Option<String>,
+    pub p: Option<String>,
+    pub sp: Option<String>,
+    pub pct: Option<String>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct Record {
+    pub row: Option<Row>,
+    pub identifiers: Option<Identifiers>,
+    pub auth_results: Option<AuthResults>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct Row {
+    pub source_ip: Option<String>,
+    pub count: Option<u64>,
+    pub policy_evaluated: Option<PolicyEvaluated>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct PolicyEvaluated {
+    pub disposition: Option<String>,
+    pub dkim: Option<String>,
+    pub spf: Option<String>,
+    pub reason: Option<Vec<Reason>>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct Reason {
+    #[serde(rename = "type")]
+    pub reason_type: Option<String>,
+    pub comment: Option<String>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct Identifiers {
+    pub header_from: Option<String>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct AuthResults {
+    pub dkim: Option<Vec<AuthDKIM>>,
+    pub spf: Option<Vec<AuthSPF>>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct AuthDKIM {
+    pub domain: Option<String>,
+    pub result: Option<String>,
+    pub selector: Option<String>,
+}
+#[derive(Debug, serde::Deserialize)]
+pub struct AuthSPF {
+    pub domain: Option<String>,
+    pub result: Option<String>,
+    pub scope: Option<String>,
+}
+
+#[derive(Template)]
+#[template(path = "dmarc_report.html")]
+pub struct DmarcReportTemplate<'a> {
+    pub report: &'a FormattedFeedback,
+}
+
+// Add this helper function to parse the DMARC XML and summarize it.
+pub fn parse_dmarc_report(xml: &str) -> Result<String, ServerError> {
+    let feedback: Feedback = xml_from_str(xml)
+        .map_err(|e| ServerError::StringError(format!("DMARC XML parse error: {}", e)))?;
+
+    let formatted_report_metadata = feedback.report_metadata.map(|meta| {
+        let date_range = meta.date_range.map(|dr| FormattedDateRange {
+            begin: match Utc.timestamp_opt(dr.begin.unwrap_or(0) as i64, 0) {
+                chrono::LocalResult::Single(d) => Some(d),
+                _ => None,
+            }
+            .map(|d| d.format("%Y-%m-%d").to_string())
+            .unwrap_or_else(|| "".to_string()),
+            end: match Utc.timestamp_opt(dr.end.unwrap_or(0) as i64, 0) {
+                chrono::LocalResult::Single(d) => Some(d),
+                _ => None,
+            }
+            .map(|d| d.format("%Y-%m-%d").to_string())
+            .unwrap_or_else(|| "".to_string()),
+        });
+        FormattedReportMetadata {
+            org_name: meta.org_name.unwrap_or_else(|| "".to_string()),
+            email: meta.email.unwrap_or_else(|| "".to_string()),
+            report_id: meta.report_id.unwrap_or_else(|| "".to_string()),
+            date_range,
+        }
+    });
+
+    let formatted_record = feedback.record.map(|records| {
+        records
+            .into_iter()
+            .map(|rec| {
+                let auth_results = rec.auth_results.map(|auth| {
+                    let dkim = auth
+                        .dkim
+                        .map(|dkims| {
+                            dkims
+                                .into_iter()
+                                .map(|d| FormattedAuthDKIM {
+                                    domain: d.domain.unwrap_or_else(|| "".to_string()),
+                                    result: d.result.unwrap_or_else(|| "".to_string()),
+                                    selector: d.selector.unwrap_or_else(|| "".to_string()),
+                                })
+                                .collect()
+                        })
+                        .unwrap_or_else(|| Vec::new());
+
+                    let spf = auth
+                        .spf
+                        .map(|spfs| {
+                            spfs.into_iter()
+                                .map(|s| FormattedAuthSPF {
+                                    domain: s.domain.unwrap_or_else(|| "".to_string()),
+                                    result: s.result.unwrap_or_else(|| "".to_string()),
+                                    scope: s.scope.unwrap_or_else(|| "".to_string()),
+                                })
+                                .collect()
+                        })
+                        .unwrap_or_else(|| Vec::new());
+
+                    FormattedAuthResults { dkim, spf }
+                });
+
+                FormattedRecord {
+                    source_ip: rec
+                        .row
+                        .as_ref()
+                        .and_then(|r| r.source_ip.clone())
+                        .unwrap_or_else(|| "".to_string()),
+                    count: rec
+                        .row
+                        .as_ref()
+                        .and_then(|r| r.count.map(|c| c.to_string()))
+                        .unwrap_or_else(|| "".to_string()),
+                    header_from: rec
+                        .identifiers
+                        .as_ref()
+                        .and_then(|i| i.header_from.clone())
+                        .unwrap_or_else(|| "".to_string()),
+                    disposition: rec
+                        .row
+                        .as_ref()
+                        .and_then(|r| r.policy_evaluated.as_ref())
+                        .and_then(|p| p.disposition.clone())
+                        .unwrap_or_else(|| "".to_string()),
+                    dkim: rec
+                        .row
+                        .as_ref()
+                        .and_then(|r| r.policy_evaluated.as_ref())
+                        .and_then(|p| p.dkim.clone())
+                        .unwrap_or_else(|| "".to_string()),
+                    spf: rec
+                        .row
+                        .as_ref()
+                        .and_then(|r| r.policy_evaluated.as_ref())
+                        .and_then(|p| p.spf.clone())
+                        .unwrap_or_else(|| "".to_string()),
+                    auth_results,
+                }
+            })
+            .collect()
+    });
+
+    let formatted_policy_published =
+        feedback
+            .policy_published
+            .map(|pol| FormattedPolicyPublished {
+                domain: pol.domain.unwrap_or_else(|| "".to_string()),
+                adkim: pol.adkim.unwrap_or_else(|| "".to_string()),
+                aspf: pol.aspf.unwrap_or_else(|| "".to_string()),
+                p: pol.p.unwrap_or_else(|| "".to_string()),
+                sp: pol.sp.unwrap_or_else(|| "".to_string()),
+                pct: pol.pct.unwrap_or_else(|| "".to_string()),
+            });
+
+    let formatted_feedback = FormattedFeedback {
+        report_metadata: formatted_report_metadata,
+        policy_published: formatted_policy_published,
+        record: formatted_record,
+    };
+
+    let template = DmarcReportTemplate {
+        report: &formatted_feedback,
+    };
+    let html = template.render()?;
+    Ok(html)
+}

server/src/templates.rs

@@ -0,0 +1,7 @@
+use askama::Template;
+
+#[derive(Template)]
+#[template(path = "dmarc_report.html")]
+pub struct DmarcReportTemplate<'a> {
+    pub feedback: &'a crate::nm::Feedback,
+}

server/src/ws.rs

@@ -0,0 +1,35 @@
+use std::{collections::HashMap, net::SocketAddr};
+
+use axum::extract::ws::{Message, WebSocket};
+use letterbox_shared::WebsocketMessage;
+use tracing::{info, warn};
+
+#[derive(Default)]
+pub struct ConnectionTracker {
+    peers: HashMap<SocketAddr, WebSocket>,
+}
+
+impl ConnectionTracker {
+    pub async fn add_peer(&mut self, socket: WebSocket, who: SocketAddr) {
+        warn!("adding {who:?} to connection tracker");
+        self.peers.insert(who, socket);
+        self.send_message_all(WebsocketMessage::RefreshMessages)
+            .await;
+    }
+    pub async fn send_message_all(&mut self, msg: WebsocketMessage) {
+        info!("send_message_all {msg}");
+        let m = serde_json::to_string(&msg).expect("failed to json encode WebsocketMessage");
+        let mut bad_peers = Vec::new();
+        for (who, socket) in &mut self.peers.iter_mut() {
+            if let Err(e) = socket.send(Message::Text(m.clone().into())).await {
+                warn!("{:?} is bad, scheduling for removal: {e}", who);
+                bad_peers.push(who.clone());
+            }
+        }
+
+        for b in bad_peers {
+            info!("removing bad peer {b:?}");
+            self.peers.remove(&b);
+        }
+    }
+}

server/templates/dmarc_report.html

@@ -0,0 +1,89 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>DMARC Report</title>
+</head>
+<body>
+    {% if report.report_metadata.is_some() %}
+        {% let meta = report.report_metadata.as_ref().unwrap() %}
+        <b>Reporter:</b> {{ meta.org_name }}<br>
+        <b>Contact:</b> {{ meta.email }}<br>
+        <b>Report ID:</b> {{ meta.report_id }}<br>
+        {% if meta.date_range.is_some() %}
+            {% let dr = meta.date_range.as_ref().unwrap() %}
+            <b>Date range:</b>
+            {{ dr.begin }}
+            to
+            {{ dr.end }}
+            <br>
+        {% endif %}
+    {% endif %}
+
+    {% if report.policy_published.is_some() %}
+        {% let pol = report.policy_published.as_ref().unwrap() %}
+        <b>Policy Published:</b>
+        <ul>
+            <li>Domain: {{ pol.domain }}</li>
+            <li>ADKIM: {{ pol.adkim }}</li>
+            <li>ASPF: {{ pol.aspf }}</li>
+            <li>Policy: {{ pol.p }}</li>
+            <li>Subdomain Policy: {{ pol.sp }}</li>
+            <li>Percent: {{ pol.pct }}</li>
+        </ul>
+    {% endif %}
+
+    {% if report.record.is_some() %}
+        <b>Records:</b>
+        <table style="border-collapse:collapse;width:100%;font-size:0.95em;">
+            <thead>
+                <tr style="background:#f0f0f0;">
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Source IP</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Count</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Header From</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Disposition</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">DKIM</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">SPF</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Auth Results</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for rec in report.record.as_ref().unwrap() %}
+                    <tr>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.source_ip }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.count }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.header_from }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.disposition }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.dkim }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.spf }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">
+                            {% if rec.auth_results.is_some() %}
+                                {% let auth = rec.auth_results.as_ref().unwrap() %}
+                                {% for dkimres in auth.dkim %}
+                                        <span style="white-space:nowrap;">
+                                            DKIM: domain=<b>{{ dkimres.domain }}</b>
+                                            selector=<b>{{ dkimres.selector }}</b>
+                                            result=<b>{{ dkimres.result }}</b>
+                                        </span><br>
+                                    {% endfor %}
+
+                                {% for spfres in auth.spf %}
+                                        <span style="white-space:nowrap;">
+                                            SPF: domain=<b>{{ spfres.domain }}</b>
+                                            scope=<b>{{ spfres.scope }}</b>
+                                            result=<b>{{ spfres.result }}</b>
+                                        </span><br>
+                                    {% endfor %}
+
+                            {% endif %}
+                        </td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    {% endif %}
+
+    {% if report.report_metadata.is_none() && report.policy_published.is_none() && report.record.is_none() %}
+        <p>No DMARC summary found.</p>
+    {% endif %}
+</body>
+</html>

shared/Cargo.toml

@@ -1,15 +1,20 @@
 [package]
 name = "letterbox-shared"
-version = "0.3.0"
-edition = "2021"
 description = "Shared module for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-build-info = "0.0.39"
-letterbox-notmuch = { version = "0.1.0", registry = "xinu" }
-serde = { version = "1.0.147", features = ["derive"] }
+build-info = "0.0.41"
+letterbox-notmuch = { path = "../notmuch", version = "0.17.28", registry = "xinu" }
+regex = "1.11.1"
+serde = { version = "1.0.219", features = ["derive"] }
+sqlx = "0.8.5"
+strum_macros = "0.27.1"
+tracing = "0.1.41"

shared/src/lib.rs

@@ -1,8 +1,14 @@
-use std::hash::{DefaultHasher, Hash, Hasher};
+use std::{
+    convert::Infallible,
+    hash::{DefaultHasher, Hash, Hasher},
+    str::FromStr,
+};
 
 use build_info::{BuildInfo, VersionControl};
 use letterbox_notmuch::SearchSummary;
+use regex::{RegexBuilder, RegexSetBuilder};
 use serde::{Deserialize, Serialize};
+use tracing::debug;
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SearchResult {
@@ -13,11 +19,20 @@ pub struct SearchResult {
     pub total: usize,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct Message {}
+#[derive(Serialize, Deserialize, Debug, strum_macros::Display)]
+pub enum WebsocketMessage {
+    RefreshMessages,
+}
 
 pub mod urls {
     pub const MOUNT_POINT: &'static str = "/api";
+    pub fn view_original(host: Option<&str>, id: &str) -> String {
+        if let Some(host) = host {
+            format!("//{host}/api/original/{id}")
+        } else {
+            format!("/api/original/{id}")
+        }
+    }
     pub fn cid_prefix(host: Option<&str>, cid: &str) -> String {
         if let Some(host) = host {
             format!("//{host}/api/cid/{cid}/")
@@ -56,3 +71,198 @@ pub fn compute_color(data: &str) -> String {
     data.hash(&mut hasher);
     format!("#{:06x}", hasher.finish() % (1 << 24))
 }
+
+#[derive(
+    Copy, Clone, Debug, Default, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize,
+)]
+pub enum MatchType {
+    From,
+    Sender,
+    To,
+    Cc,
+    Subject,
+    ListId,
+    DeliveredTo,
+    XForwardedTo,
+    ReplyTo,
+    XOriginalTo,
+    XSpam,
+    Body,
+    #[default]
+    Unknown,
+}
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct Match {
+    pub match_type: MatchType,
+    pub needle: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct Rule {
+    pub stop_on_match: bool,
+    pub matches: Vec<Match>,
+    pub tag: String,
+}
+impl Rule {
+    pub fn is_match(&self, header_key: &str, header_value: &str) -> bool {
+        let pats: Vec<_> = self
+            .matches
+            .iter()
+            .filter_map(|m| match m.match_type {
+                MatchType::To => Some("^(to|cc|bcc|x-original-to)$"),
+                MatchType::From => Some("^from$"),
+                MatchType::Sender => Some("^sender$"),
+                MatchType::Subject => Some("^subject$"),
+                MatchType::ListId => Some("^list-id$"),
+                MatchType::XOriginalTo => Some("^x-original-to$"),
+                MatchType::ReplyTo => Some("^reply-to$"),
+                MatchType::XSpam => Some("^x-spam$"),
+                MatchType::Body => None,
+                c => panic!("TODO handle '{c:?}' match type"),
+            })
+            .collect();
+
+        let set = RegexSetBuilder::new(&pats)
+            .case_insensitive(true)
+            .build()
+            .expect("failed to compile regex for matches");
+        let matches: Vec<_> = set.matches(header_key).into_iter().collect();
+        if !matches.is_empty() {
+            //info!("matched key '{header_key}' '{header_value}'");
+            for m_idx in matches {
+                let needle = regex::escape(&self.matches[m_idx].needle);
+                let pat = RegexBuilder::new(&needle)
+                    .case_insensitive(true)
+                    .build()
+                    .expect("failed to compile regex for needle");
+                if pat.is_match(header_value) {
+                    debug!("{header_key} matched {header_value} against {needle}");
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}
+
+mod matches {
+    // From https://linux.die.net/man/5/procmailrc
+    // If the regular expression contains '^TO_' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^-a-zA-Z0-9_.])?)'
+    // If the regular expression contains '^TO' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^a-zA-Z])?)', which should catch all destination specifications containing a specific word.
+
+    pub const TO: &'static str = "TO";
+    pub const CC: &'static str = "Cc";
+    pub const TOCC: &'static str = "(TO|Cc)";
+    pub const FROM: &'static str = "From";
+    pub const SENDER: &'static str = "Sender";
+    pub const SUBJECT: &'static str = "Subject";
+    pub const DELIVERED_TO: &'static str = "Delivered-To";
+    pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
+    pub const REPLY_TO: &'static str = "Reply-To";
+    pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
+    pub const LIST_ID: &'static str = "List-ID";
+    pub const X_SPAM: &'static str = "X-Spam";
+    pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
+}
+
+impl FromStr for Match {
+    type Err = Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        // Examples:
+        //   "* 1^0 ^TOsonyrewards.com@xinu.tv"
+        //   "* ^TOsonyrewards.com@xinu.tv"
+        let mut it = s.split_whitespace().skip(1);
+        let mut needle = it.next().unwrap();
+        if needle == "1^0" {
+            needle = it.next().unwrap();
+        }
+        let mut needle = vec![needle];
+        needle.extend(it);
+        let needle = needle.join(" ");
+        let first = needle.chars().nth(0).unwrap_or(' ');
+        use matches::*;
+        if first == '^' {
+            let needle = &needle[1..];
+            if needle.starts_with(TO) {
+                return Ok(Match {
+                    match_type: MatchType::To,
+                    needle: cleanup_match(TO, needle),
+                });
+            } else if needle.starts_with(FROM) {
+                return Ok(Match {
+                    match_type: MatchType::From,
+                    needle: cleanup_match(FROM, needle),
+                });
+            } else if needle.starts_with(CC) {
+                return Ok(Match {
+                    match_type: MatchType::Cc,
+                    needle: cleanup_match(CC, needle),
+                });
+            } else if needle.starts_with(TOCC) {
+                return Ok(Match {
+                    match_type: MatchType::To,
+                    needle: cleanup_match(TOCC, needle),
+                });
+            } else if needle.starts_with(SENDER) {
+                return Ok(Match {
+                    match_type: MatchType::Sender,
+                    needle: cleanup_match(SENDER, needle),
+                });
+            } else if needle.starts_with(SUBJECT) {
+                return Ok(Match {
+                    match_type: MatchType::Subject,
+                    needle: cleanup_match(SUBJECT, needle),
+                });
+            } else if needle.starts_with(X_ORIGINAL_TO) {
+                return Ok(Match {
+                    match_type: MatchType::XOriginalTo,
+                    needle: cleanup_match(X_ORIGINAL_TO, needle),
+                });
+            } else if needle.starts_with(LIST_ID) {
+                return Ok(Match {
+                    match_type: MatchType::ListId,
+                    needle: cleanup_match(LIST_ID, needle),
+                });
+            } else if needle.starts_with(REPLY_TO) {
+                return Ok(Match {
+                    match_type: MatchType::ReplyTo,
+                    needle: cleanup_match(REPLY_TO, needle),
+                });
+            } else if needle.starts_with(X_SPAM_FLAG) {
+                return Ok(Match {
+                    match_type: MatchType::XSpam,
+                    needle: '*'.to_string(),
+                });
+            } else if needle.starts_with(X_SPAM) {
+                return Ok(Match {
+                    match_type: MatchType::XSpam,
+                    needle: '*'.to_string(),
+                });
+            } else if needle.starts_with(DELIVERED_TO) {
+                return Ok(Match {
+                    match_type: MatchType::DeliveredTo,
+                    needle: cleanup_match(DELIVERED_TO, needle),
+                });
+            } else if needle.starts_with(X_FORWARDED_TO) {
+                return Ok(Match {
+                    match_type: MatchType::XForwardedTo,
+                    needle: cleanup_match(X_FORWARDED_TO, needle),
+                });
+            } else {
+                unreachable!("needle: '{needle}'")
+            }
+        } else {
+            return Ok(Match {
+                match_type: MatchType::Body,
+                needle: cleanup_match("", &needle),
+            });
+        }
+    }
+}
+fn unescape(s: &str) -> String {
+    s.replace('\\', "")
+}
+pub fn cleanup_match(prefix: &str, s: &str) -> String {
+    unescape(&s[prefix.len()..]).replace(".*", "")
+}

web/Cargo.toml

@@ -1,52 +1,58 @@
 [package]
-version = "0.1.0"
 name = "letterbox-web"
-authors = ["Bill Thiede <git@xinu.tv>"]
-edition = "2021"
 description = "Web frontend for letterbox"
-license = "UNLICENSED"
-repository = "https://git.z.xinu.tv/wathiede/letterbox"
-publish = ["xinu"]
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+version.workspace = true
 
 [build-dependencies]
-build-info-build = "0.0.39"
+build-info-build = "0.0.41"
 
 [dev-dependencies]
-wasm-bindgen-test = "0.3.33"
+wasm-bindgen-test = "0.3.50"
 
 [dependencies]
 console_error_panic_hook = "0.1.7"
-log = "0.4.17"
+log = "0.4.27"
 seed = { version = "0.10.0", features = ["routing"] }
 #seed = "0.9.2"
-console_log = { version = "0.1.0", registry = "xinu" }
-serde = { version = "1.0.147", features = ["derive"] }
+console_log = { version = "0.1.4", registry = "xinu" }
+serde = { version = "1.0.219", features = ["derive"] }
 itertools = "0.14.0"
-serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
-chrono = "0.4.31"
+serde_json = { version = "1.0.140", features = ["unbounded_depth"] }
+chrono = "0.4.40"
 graphql_client = "0.14.0"
-thiserror = "2.0.0"
-seed_hooks = { git = "https://github.com/wathiede/styles_hooks", package = "seed_hooks", branch = "main" }
+thiserror = "2.0.12"
 gloo-net = { version = "0.6.0", features = ["json", "serde_json"] }
 human_format = "1.1.0"
-build-info = "0.0.39"
+build-info = "0.0.41"
 wasm-bindgen = "=0.2.100"
-uuid = { version = "1.13.1", features = [
+uuid = { version = "1.16.0", features = [
   "js",
 ] } # direct dep to set js feature, prevents Rng issues
-letterbox-shared = { version = "0.1.0", registry = "xinu" }
-letterbox-notmuch = { version = "0.1.0", path = "../notmuch", registry = "xinu" }
+letterbox-shared = { path = "../shared/", version = "0.17.28", registry = "xinu" }
+seed_hooks = { version = "0.4.1", registry = "xinu" }
+strum_macros = "0.27.1"
+gloo-console = "0.3.0"
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-sockets = "1.0.0"
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = ['-Os']
 
 [dependencies.web-sys]
-version = "0.3.58"
+version = "0.3.77"
 features = [
   "Clipboard",
   "DomRect",
   "Element",
+  "History",
   "MediaQueryList",
   "Navigator",
+  "Performance",
+  "ScrollRestoration",
   "Window",
 ]

web/Trunk.toml

@@ -6,9 +6,16 @@ release = false
 address = "0.0.0.0"
 port = 6758
 
+[[proxy]]
+ws = true
+backend = "ws://localhost:9345/api/ws"
+
 [[proxy]]
 backend = "http://localhost:9345/api/"
 
+[[proxy]]
+backend = "http://localhost:9345/notification/"
+
 [[hooks]]
 stage = "pre_build"
 command = "printf"

web/graphql/catchup.graphql

@@ -0,0 +1,3 @@
+query CatchupQuery($query: String!) {
+    catchup(query: $query)
+}

web/graphql/schema.json

@@ -671,6 +671,30 @@
                 }
               }
             },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "xOriginalTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
+            {
+              "args": [],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "deliveredTo",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Email",
+                "ofType": null
+              }
+            },
             {
               "args": [],
               "deprecationReason": null,
@@ -1268,6 +1292,45 @@
                 }
               }
             },
+            {
+              "args": [
+                {
+                  "defaultValue": null,
+                  "description": null,
+                  "name": "query",
+                  "type": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              ],
+              "deprecationReason": null,
+              "description": null,
+              "isDeprecated": false,
+              "name": "catchup",
+              "type": {
+                "kind": "NON_NULL",
+                "name": null,
+                "ofType": {
+                  "kind": "LIST",
+                  "name": null,
+                  "ofType": {
+                    "kind": "NON_NULL",
+                    "name": null,
+                    "ofType": {
+                      "kind": "SCALAR",
+                      "name": "String",
+                      "ofType": null
+                    }
+                  }
+                }
+              }
+            },
             {
               "args": [
                 {

web/graphql/show_thread.graphql

@@ -31,6 +31,14 @@ query ShowThreadQuery($threadId: String!) {
           name
           addr
         }
+        xOriginalTo {
+          name
+          addr
+        }
+        deliveredTo {
+          name
+          addr
+        }
         timestamp
         body {
           __typename

web/index.html

@@ -16,10 +16,11 @@
   <link data-trunk rel="css" href="static/vars.css" />
   <link data-trunk rel="tailwind-css" href="./src/tailwind.css" />
   <link data-trunk rel="css" href="static/overrides.css" />
+  <link data-trunk rel="css" href="static/email-specific.css" />
 </head>
 
 <body>
   <section id="app"></section>
 </body>
 
-</html>
+</html>

web/src/graphql.rs

@@ -12,6 +12,14 @@ use serde::{de::DeserializeOwned, Serialize};
 )]
 pub struct FrontPageQuery;
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "graphql/schema.json",
+    query_path = "graphql/catchup.graphql",
+    response_derives = "Debug"
+)]
+pub struct CatchupQuery;
+
 #[derive(GraphQLQuery)]
 #[graphql(
     schema_path = "graphql/schema.json",

web/src/main.rs

@@ -11,6 +11,7 @@ mod consts;
 mod graphql;
 mod state;
 mod view;
+mod websocket;
 
 fn main() {
     // This provides better error messages in debug mode.
@@ -18,6 +19,9 @@ fn main() {
     #[cfg(debug_assertions)]
     console_error_panic_hook::set_once();
 
+    #[cfg(debug_assertions)]
+    let lvl = Level::Debug;
+    #[cfg(not(debug_assertions))]
     let lvl = Level::Info;
     console_log::init_with_level(lvl).expect("failed to initialize console logging");
     // Mount the `app` to the element with the `id` "app".

web/src/state.rs

@@ -1,6 +1,7 @@
 use std::collections::HashSet;
 
 use graphql_client::GraphQLQuery;
+use letterbox_shared::WebsocketMessage;
 use log::{debug, error, info, warn};
 use seed::{prelude::*, *};
 use thiserror::Error;
@@ -11,6 +12,7 @@ use crate::{
     consts::SEARCH_RESULTS_PER_PAGE,
     graphql,
     graphql::{front_page_query::*, send_graphql, show_thread_query::*},
+    websocket,
 };
 
 /// Used to fake the unread string while in development
@@ -29,16 +31,21 @@ pub fn unread_query() -> &'static str {
 pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
     let version = letterbox_shared::build_version(bi);
     info!("Build Info: {}", version);
+    // Disable restoring to scroll position when navigating
+    window()
+        .history()
+        .expect("couldn't get history")
+        .set_scroll_restoration(web_sys::ScrollRestoration::Manual)
+        .expect("failed to set scroll restoration to manual");
     if url.hash().is_none() {
         orders.request_url(urls::search(unread_query(), 0));
     } else {
-        orders.notify(subs::UrlRequested::new(url));
+        orders.request_url(url.clone());
     };
-    orders.stream(streams::window_event(Ev::Resize, |_| Msg::OnResize));
     // TODO(wathiede): only do this while viewing the index? Or maybe add a new message that force
     // 'notmuch new' on the server periodically?
-    orders.stream(streams::interval(30_000, || Msg::RefreshStart));
-    orders.subscribe(on_url_changed);
+    //orders.stream(streams::interval(30_000, || Msg::RefreshStart));
+    orders.subscribe(Msg::OnUrlChanged);
     orders.stream(streams::window_event(Ev::Scroll, |_| Msg::WindowScrolled));
 
     build_info::build_info!(fn bi);
@@ -53,18 +60,20 @@ pub fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
             client: version,
             server: None,
         },
+        catchup: None,
+        last_url: Url::current(),
+        websocket: websocket::init("/api/ws", &mut orders.proxy(Msg::WebSocket)),
     }
 }
 
-fn on_url_changed(uc: subs::UrlChanged) -> Msg {
-    let mut url = uc.0;
-    info!(
-        "url changed '{}', history {}",
-        url,
-        history().length().unwrap_or(0)
-    );
-    let hpp = url.remaining_hash_path_parts();
-    match hpp.as_slice() {
+fn on_url_changed(old: &Url, mut new: Url) -> Msg {
+    let did_change = *old != new;
+    let mut messages = Vec::new();
+    if did_change {
+        messages.push(Msg::ScrollToTop)
+    }
+    let hpp = new.remaining_hash_path_parts();
+    let msg = match hpp.as_slice() {
         ["t", tid] => Msg::ShowThreadRequest {
             thread_id: tid.to_string(),
         },
@@ -101,11 +110,14 @@ fn on_url_changed(uc: subs::UrlChanged) -> Msg {
                 last: None,
             }
         }
-    }
+    };
+    messages.push(msg);
+    Msg::MultiMsg(messages)
 }
 
 // `update` describes how to handle each `Msg`.
 pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
+    info!("update({})", msg);
     match msg {
         Msg::Noop => {}
         Msg::RefreshStart => {
@@ -131,7 +143,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             orders.perform_cmd(async move { Msg::Refresh });
         }
         Msg::Refresh => {
-            orders.perform_cmd(async move { on_url_changed(subs::UrlChanged(Url::current())) });
+            orders.request_url(Url::current());
         }
         Msg::Reload => {
             window()
@@ -139,7 +151,10 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 .reload()
                 .expect("failed to reload window");
         }
-        Msg::OnResize => (),
+        Msg::OnUrlChanged(new_url) => {
+            orders.send_msg(on_url_changed(&model.last_url, new_url.0.clone()));
+            model.last_url = new_url.0;
+        }
 
         Msg::NextPage => {
             match &model.context {
@@ -181,10 +196,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             };
         }
         Msg::GoToSearchResults => {
-            let url = urls::search(&model.query, 0);
-            info!("GoToSearchRestuls Start");
-            orders.request_url(url);
-            info!("GoToSearchRestuls End");
+            orders.send_msg(Msg::SearchQuery(model.query.clone()));
         }
 
         Msg::UpdateQuery(query) => model.query = query,
@@ -279,7 +291,9 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                 )
             });
         }
-        Msg::FrontPageResult(Err(e)) => error!("error FrontPageResult: {e:?}"),
+        Msg::FrontPageResult(Err(e)) => {
+            error!("error FrontPageResult: {e:?}");
+        }
         Msg::FrontPageResult(Ok(graphql_client::Response {
             data: None,
             errors: None,
@@ -307,7 +321,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     })
                     .collect(),
             );
-            info!("pager {:#?}", data.search.page_info);
             let selected_threads = 'context: {
                 if let Context::SearchResult {
                     results,
@@ -390,6 +403,38 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
         Msg::ShowThreadResult(bad) => {
             error!("show_thread_query error: {bad:#?}");
         }
+        Msg::CatchupRequest { query } => {
+            orders.perform_cmd(async move {
+                Msg::CatchupResult(
+                    send_graphql::<_, graphql::catchup_query::ResponseData>(
+                        graphql::CatchupQuery::build_query(graphql::catchup_query::Variables {
+                            query,
+                        }),
+                    )
+                    .await,
+                )
+            });
+        }
+        Msg::CatchupResult(Ok(graphql_client::Response {
+            data: Some(data), ..
+        })) => {
+            let items = data.catchup;
+            if items.is_empty() {
+                orders.send_msg(Msg::GoToSearchResults);
+                model.catchup = None;
+            } else {
+                orders.request_url(urls::thread(&items[0]));
+                model.catchup = Some(Catchup {
+                    items: items
+                        .into_iter()
+                        .map(|id| CatchupItem { id, seen: false })
+                        .collect(),
+                });
+            }
+        }
+        Msg::CatchupResult(bad) => {
+            error!("catchup_query error: {bad:#?}");
+        }
         Msg::SelectionSetNone => {
             if let Context::SearchResult {
                 selected_threads, ..
@@ -503,8 +548,11 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .expect("failed to copy to clipboard");
             });
         }
+        Msg::ScrollToTop => {
+            web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
+        }
         Msg::WindowScrolled => {
-            info!("WindowScrolled");
+            // TODO: model.content_el doesn't go to None like it should when a DOM is recreated and the refrenced element goes away
             if let Some(el) = model.content_el.get() {
                 let ih = window()
                     .inner_height()
@@ -513,7 +561,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .value_of();
 
                 let r = el.get_bounding_client_rect();
-                info!("r {r:?} ih {ih}");
                 if r.height() < ih {
                     // The whole content fits in the window, no scrollbar
                     orders.send_msg(Msg::SetProgress(0.));
@@ -545,7 +592,8 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             model.read_completion_ratio = ratio;
         }
         Msg::UpdateServerVersion(version) => {
-            if version != model.versions.client {
+            // Only git versions contain dash, don't autoreload there
+            if !version.contains('-') && version != model.versions.client {
                 warn!(
                     "Server ({}) and client ({}) version mismatch, reloading",
                     version, model.versions.client
@@ -554,8 +602,124 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             }
             model.versions.server = Some(version);
         }
+
+        Msg::CatchupStart => {
+            let query = if model.query.contains("is:unread") {
+                model.query.to_string()
+            } else {
+                format!("{} is:unread", model.query)
+            };
+            info!("starting catchup mode w/ {}", query);
+            orders.send_msg(Msg::ScrollToTop);
+            orders.send_msg(Msg::CatchupRequest { query });
+        }
+        Msg::CatchupKeepUnread => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                if let Context::ThreadResult {
+                    thread:
+                        ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                            messages,
+                            ..
+                        }),
+                    ..
+                } = &model.context
+                {
+                    //orders.send_msg(Msg::SetUnread(thread_id, false));
+                    let unread_messages: Vec<_> = messages
+                        .iter()
+                        .filter(|msg| msg.tags.iter().any(|t| t == "unread"))
+                        .map(|msg| &msg.id)
+                        .collect();
+                    if unread_messages.is_empty() {
+                        // All messages are read, so mark them all unread
+                        orders.send_msg(Msg::SetUnread(thread_id, true));
+                    } else {
+                        // Do nothing if there are some messages unread
+                    }
+                } else {
+                    // News post, not email, just mark unread
+                    orders.send_msg(Msg::SetUnread(thread_id, true));
+                };
+            } else {
+                // This shouldn't happen
+                warn!("no current thread_id");
+            }
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupMarkAsRead => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                orders.send_msg(Msg::SetUnread(thread_id, false));
+            };
+            orders.send_msg(Msg::CatchupNext);
+        }
+        Msg::CatchupNext => {
+            orders.send_msg(Msg::ScrollToTop);
+            let Some(catchup) = &mut model.catchup else {
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            let Some(thread_id) = current_thread_id(&model.context) else {
+                return;
+            };
+            let Some(idx) = catchup
+                .items
+                .iter()
+                .inspect(|i| info!("i {i:?} thread_id {thread_id}"))
+                .position(|i| i.id == thread_id)
+            else {
+                // All items have been seen
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+            catchup.items[idx].seen = true;
+            if idx < catchup.items.len() - 1 {
+                // Reached last item
+                orders.request_url(urls::thread(&catchup.items[idx + 1].id));
+                return;
+            } else {
+                orders.send_msg(Msg::CatchupExit);
+                orders.send_msg(Msg::GoToSearchResults);
+                return;
+            };
+        }
+        Msg::CatchupExit => {
+            orders.send_msg(Msg::ScrollToTop);
+            model.catchup = None;
+        }
+
+        Msg::WebSocket(ws) => {
+            websocket::update(ws, &mut model.websocket, &mut orders.proxy(Msg::WebSocket));
+            while let Some(msg) = model.websocket.updates.pop_front() {
+                orders.send_msg(Msg::WebsocketMessage(msg));
+            }
+        }
+        Msg::WebsocketMessage(msg) => {
+            match msg {
+                WebsocketMessage::RefreshMessages => orders.send_msg(Msg::Refresh),
+            };
+        }
     }
 }
+
+fn current_thread_id(context: &Context) -> Option<String> {
+    match context {
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                    thread_id, ..
+                }),
+            ..
+        } => Some(thread_id.clone()),
+        Context::ThreadResult {
+            thread:
+                ShowThreadQueryThread::NewsPost(ShowThreadQueryThreadOnNewsPost { thread_id, .. }),
+            ..
+        } => Some(thread_id.clone()),
+        _ => None,
+    }
+}
+
 // `Model` describes our app state.
 pub struct Model {
     pub query: String,
@@ -565,6 +729,9 @@ pub struct Model {
     pub read_completion_ratio: f64,
     pub content_el: ElRef<HtmlElement>,
     pub versions: Version,
+    pub catchup: Option<Catchup>,
+    pub last_url: Url,
+    pub websocket: websocket::Model,
 }
 
 #[derive(Debug)]
@@ -601,6 +768,16 @@ pub enum Context {
     },
 }
 
+pub struct Catchup {
+    pub items: Vec<CatchupItem>,
+}
+
+#[derive(Debug)]
+pub struct CatchupItem {
+    pub id: String,
+    pub seen: bool,
+}
+
 pub struct Tag {
     pub name: String,
     pub bg_color: String,
@@ -614,20 +791,22 @@ pub enum RefreshingState {
     Error(String),
 }
 // `Msg` describes the different events you can modify state with.
+#[derive(strum_macros::Display)]
 pub enum Msg {
     Noop,
     // Tell the client to refresh its state
     Refresh,
     // Tell the client to reload whole page from server
     Reload,
-    // Window has changed size
-    OnResize,
+    // TODO: add GoToUrl
+    OnUrlChanged(subs::UrlChanged),
     // Tell the server to update state
     RefreshStart,
     RefreshDone(Option<gloo_net::Error>),
     NextPage,
     PreviousPage,
     GoToSearchResults,
+
     UpdateQuery(String),
     SearchQuery(String),
 
@@ -651,10 +830,14 @@ pub enum Msg {
     ShowThreadResult(
         Result<graphql_client::Response<graphql::show_thread_query::ResponseData>, gloo_net::Error>,
     ),
+    CatchupRequest {
+        query: String,
+    },
+    CatchupResult(
+        Result<graphql_client::Response<graphql::catchup_query::ResponseData>, gloo_net::Error>,
+    ),
 
-    #[allow(dead_code)]
     SelectionSetNone,
-    #[allow(dead_code)]
     SelectionSetAll,
     SelectionAddTag(String),
     #[allow(dead_code)]
@@ -670,7 +853,17 @@ pub enum Msg {
 
     CopyToClipboard(String),
 
+    ScrollToTop,
     WindowScrolled,
     SetProgress(f64),
     UpdateServerVersion(String),
+
+    CatchupStart,
+    CatchupKeepUnread,
+    CatchupMarkAsRead,
+    CatchupNext,
+    CatchupExit,
+
+    WebSocket(websocket::Msg),
+    WebsocketMessage(WebsocketMessage),
 }

web/src/websocket.rs

@@ -0,0 +1,213 @@
+use std::{collections::VecDeque, rc::Rc};
+
+use letterbox_shared::WebsocketMessage;
+use log::{debug, error};
+use seed::prelude::*;
+use serde::{Deserialize, Serialize};
+#[cfg(not(target_arch = "wasm32"))]
+#[allow(dead_code)]
+mod wasm_sockets {
+    use std::{cell::RefCell, rc::Rc};
+
+    use thiserror::Error;
+    use web_sys::{CloseEvent, ErrorEvent};
+
+    #[derive(Debug)]
+    pub struct JsValue;
+    #[derive(Debug)]
+    pub enum ConnectionStatus {
+        /// Connecting to a server
+        Connecting,
+        /// Connected to a server
+        Connected,
+        /// Disconnected from a server due to an error
+        Error,
+        /// Disconnected from a server without an error
+        Disconnected,
+    }
+    #[derive(Debug)]
+    pub struct EventClient {
+        pub status: Rc<RefCell<ConnectionStatus>>,
+    }
+    impl EventClient {
+        pub fn new(_: &str) -> Result<Self, WebSocketError> {
+            todo!("this is a mock")
+        }
+        pub fn send_string(&self, _essage: &str) -> Result<(), JsValue> {
+            todo!("this is a mock")
+        }
+        pub fn set_on_error(&mut self, _: Option<Box<dyn Fn(ErrorEvent)>>) {
+            todo!("this is a mock")
+        }
+        pub fn set_on_connection(&mut self, _: Option<Box<dyn Fn(&EventClient)>>) {
+            todo!("this is a mock")
+        }
+        pub fn set_on_close(&mut self, _: Option<Box<dyn Fn(CloseEvent)>>) {
+            todo!("this is a mock")
+        }
+        pub fn set_on_message(&mut self, _: Option<Box<dyn Fn(&EventClient, Message)>>) {
+            todo!("this is a mock")
+        }
+    }
+    #[derive(Debug, Clone)]
+    pub enum Message {
+        Text(String),
+        Binary(Vec<u8>),
+    }
+    #[derive(Debug, Clone, Error)]
+    pub enum WebSocketError {}
+}
+#[cfg(not(target_arch = "wasm32"))]
+use wasm_sockets::{ConnectionStatus, EventClient, Message, WebSocketError};
+#[cfg(target_arch = "wasm32")]
+use wasm_sockets::{ConnectionStatus, EventClient, Message, WebSocketError};
+use web_sys::CloseEvent;
+
+/// Message from the client to the server.
+#[derive(Serialize, Deserialize)]
+pub struct ClientMessage {
+    pub text: String,
+}
+
+//const WS_URL: &str = "wss://9000.z.xinu.tv/api/ws";
+//const WS_URL: &str = "wss://9345.z.xinu.tv/api/graphql/ws";
+//const WS_URL: &str = "wss://6758.z.xinu.tv/api/ws";
+
+// ------ ------
+//     Model
+// ------ ------
+
+pub struct Model {
+    ws_url: String,
+    web_socket: EventClient,
+    web_socket_reconnector: Option<StreamHandle>,
+    pub updates: VecDeque<WebsocketMessage>,
+}
+
+// ------ ------
+//     Init
+// ------ ------
+
+pub fn init(ws_url: &str, orders: &mut impl Orders<Msg>) -> Model {
+    Model {
+        ws_url: ws_url.to_string(),
+        web_socket: create_websocket(ws_url, orders).unwrap(),
+        web_socket_reconnector: None,
+        updates: VecDeque::new(),
+    }
+}
+
+// ------ ------
+//    Update
+// ------ ------
+
+pub enum Msg {
+    WebSocketOpened,
+    TextMessageReceived(WebsocketMessage),
+    WebSocketClosed(CloseEvent),
+    WebSocketFailed,
+    ReconnectWebSocket(usize),
+    #[allow(dead_code)]
+    SendMessage(ClientMessage),
+}
+
+pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
+    match msg {
+        Msg::WebSocketOpened => {
+            model.web_socket_reconnector = None;
+            debug!("WebSocket connection is open now");
+        }
+        Msg::TextMessageReceived(msg) => {
+            model.updates.push_back(msg);
+        }
+        Msg::WebSocketClosed(close_event) => {
+            debug!(
+                r#"==================
+WebSocket connection was closed:
+Clean: {0}
+Code: {1}
+Reason: {2}
+=================="#,
+                close_event.was_clean(),
+                close_event.code(),
+                close_event.reason()
+            );
+
+            // Chrome doesn't invoke `on_error` when the connection is lost.
+            if !close_event.was_clean() && model.web_socket_reconnector.is_none() {
+                model.web_socket_reconnector = Some(
+                    orders.stream_with_handle(streams::backoff(None, Msg::ReconnectWebSocket)),
+                );
+            }
+        }
+        Msg::WebSocketFailed => {
+            debug!("WebSocket failed");
+            if model.web_socket_reconnector.is_none() {
+                model.web_socket_reconnector = Some(
+                    orders.stream_with_handle(streams::backoff(None, Msg::ReconnectWebSocket)),
+                );
+            }
+        }
+        Msg::ReconnectWebSocket(retries) => {
+            debug!("Reconnect attempt: {}", retries);
+            model.web_socket = create_websocket(&model.ws_url, orders).unwrap();
+        }
+        Msg::SendMessage(msg) => {
+            let txt = serde_json::to_string(&msg).unwrap();
+            model.web_socket.send_string(&txt).unwrap();
+        }
+    }
+}
+
+fn create_websocket(url: &str, orders: &impl Orders<Msg>) -> Result<EventClient, WebSocketError> {
+    let msg_sender = orders.msg_sender();
+
+    let mut client = EventClient::new(url)?;
+
+    client.set_on_error(Some(Box::new(|error| {
+        gloo_console::error!("WS: ", error);
+    })));
+
+    let send = msg_sender.clone();
+    client.set_on_connection(Some(Box::new(move |client: &EventClient| {
+        debug!("{:#?}", client.status);
+        let msg = match *client.status.borrow() {
+            ConnectionStatus::Connecting => {
+                debug!("Connecting...");
+                None
+            }
+            ConnectionStatus::Connected => Some(Msg::WebSocketOpened),
+            ConnectionStatus::Error => Some(Msg::WebSocketFailed),
+            ConnectionStatus::Disconnected => {
+                debug!("Disconnected");
+                None
+            }
+        };
+        send(msg);
+    })));
+
+    let send = msg_sender.clone();
+    client.set_on_close(Some(Box::new(move |ev| {
+        debug!("WS: Connection closed");
+        send(Some(Msg::WebSocketClosed(ev)));
+    })));
+
+    let send = msg_sender.clone();
+    client.set_on_message(Some(Box::new(move |_: &EventClient, msg: Message| {
+        decode_message(msg, Rc::clone(&send))
+    })));
+
+    Ok(client)
+}
+
+fn decode_message(message: Message, msg_sender: Rc<dyn Fn(Option<Msg>)>) {
+    match message {
+        Message::Text(txt) => {
+            let msg: WebsocketMessage = serde_json::from_str(&txt).unwrap_or_else(|e| {
+                panic!("failed to parse json into WebsocketMessage: {e}\n'{txt}'")
+            });
+            msg_sender(Some(Msg::TextMessageReceived(msg)));
+        }
+        m => error!("unexpected message type received of {m:?}"),
+    }
+}

web/static/email-specific.css

@@ -0,0 +1,11 @@
+.mail-thread .content.from-noreply-news-bloomberg-com a {
+  background-color: initial !important;
+}
+
+.mail-thread .content.from-noreply-news-bloomberg-com h2 {
+  margin: 0 !important;
+  padding: 0 !important;
+}
+.mail-thread .content.from-dmarcreport-microsoft-com div {
+  font-family:  ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji" !important;
+}

web/static/overrides.css

@@ -1,69 +1,80 @@
 html {
-    background-color: black;
+  background-color: black;
 }
 
-.mail-thread a,
+.mail-thread .content a,
 .news-post a {
-    color: var(--color-link) !important;
-    text-decoration: underline;
+  color: var(--color-link) !important;
+  text-decoration: underline;
 }
 
-.mail-thread br,
+.mail-thread .content br,
 .news-post br {
-    display: block;
-    margin-top: 1em;
-    content: " ";
+  display: block;
+  margin-top: 1em;
+  content: " ";
 }
 
-.mail-thread h1,
-.mail-thread h2,
-.mail-thread h3,
-.mail-thread h4,
+.mail-thread .content h1,
+.mail-thread .content h2,
+.mail-thread .content h3,
+.mail-thread .content h4,
 .news-post h1,
 .news-post h2,
 .news-post h3,
 .news-post h4 {
-    margin-top: 1em !important;
-    margin-bottom: 1em !important;
+  margin-top: 1em !important;
+  margin-bottom: 1em !important;
 }
 
-.mail-thread p,
+.mail-thread .content p,
 .news-post p {
-    margin-bottom: 1em;
+  margin-bottom: 1em;
 }
 
-.mail-thread pre,
-.mail-thread code,
-.news-post pre,
+.mail-thread .content pre,
+.news-post pre {
+  font-family: monospace;
+  background-color: #eee !important;
+  padding: 0.5em;
+  white-space: break-spaces;
+}
+
+.mail-thread .content code,
 .news-post code {
-    font-family: monospace;
-    background-color: #eee !important;
-    padding: 0.5em !important;
+  font-family: monospace;
+  white-space: break-spaces;
+  background-color: #eee !important;
 }
 
-.mail-thread blockquote {
-    padding-left: 1em;
-    border-left: 2px solid #ddd;
+.mail-thread .content blockquote {
+  padding-left: 1em;
+  border-left: 2px solid #ddd;
 }
 
-.mail-thread ol,
-.mail-thread ul {
-    margin-left: 2em;
+.mail-thread .content ol,
+.mail-thread .content ul {
+  margin-left: 2em;
 }
 
 /* Hackaday figures have unreadable black on dark grey */
 .news-post figcaption.wp-caption-text {
-    background-color: initial !important;
+  background-color: initial !important;
 }
 
 .news-post.site-nautilus .article-ad,
 .news-post.site-nautilus .primis-ad {
-    display: none !important;
+  display: none !important;
+}
+
+.news-post.site-seiya-me figure>pre,
+.news-post.site-seiya-me figure>pre>code {
+  background-color: black !important;
 }
 
 .news-post.site-slashdot .story-byline {
-    display: block !important;
-    height: initial !important;
-    overflow: auto !important;
-    position: static !important;
-}
+  display: block !important;
+  height: initial !important;
+  overflow: auto !important;
+  position: static !important;
+}