chore: Release

.gitea/workflows/rust.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-            toolchain: stable
+            toolchain: nightly
             target: wasm32-unknown-unknown
       - run: cargo install trunk
       - run: cd web; trunk build

Cargo.toml

@@ -8,7 +8,7 @@ authors = ["Bill Thiede <git@xinu.tv>"]
 edition = "2021"
 license = "UNLICENSED"
 publish = ["xinu"]
-version = "0.15.11"
+version = "0.17.34"
 repository = "https://git.z.xinu.tv/wathiede/letterbox"
 
 [profile.dev]

notmuch/Cargo.toml

@@ -11,14 +11,14 @@ version.workspace = true
 
 
 [dependencies]
-log = "0.4.14"
+log = "0.4.27"
-mailparse = "0.16.0"
+mailparse = "0.16.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["unbounded_depth"] }
-thiserror = "2.0.0"
+thiserror = "2.0.12"
 tracing = "0.1.41"
 
 [dev-dependencies]
 itertools = "0.14.0"
 pretty_assertions = "1"
-rayon = "1.5"
+rayon = "1.10"

notmuch/src/lib.rs

@@ -214,9 +214,8 @@ use std::{
     process::Command,
 };
 
-use log::{error, info};
 use serde::{Deserialize, Serialize};
-use tracing::instrument;
+use tracing::{error, info, instrument, warn};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -503,15 +502,28 @@ impl Notmuch {
         self.tags_for_query("*")
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_add(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
-        self.run_notmuch(["tag", &format!("+{tag}"), search_term])?;
+        self.tags_add(tag, &[search_term])
+    }
+
+    #[instrument(skip_all, fields(tag=tag,search_term=?search_term))]
+    pub fn tags_add(&self, tag: &str, search_term: &[&str]) -> Result<(), NotmuchError> {
+        let tag = format!("+{tag}");
+        let mut args = vec!["tag", &tag];
+        args.extend(search_term);
+        self.run_notmuch(&args)?;
         Ok(())
     }
 
-    #[instrument(skip_all, fields(tag=tag,search_term=search_term))]
     pub fn tag_remove(&self, tag: &str, search_term: &str) -> Result<(), NotmuchError> {
-        self.run_notmuch(["tag", &format!("-{tag}"), search_term])?;
+        self.tags_remove(tag, &[search_term])
+    }
+    #[instrument(skip_all, fields(tag=tag,search_term=?search_term))]
+    pub fn tags_remove(&self, tag: &str, search_term: &[&str]) -> Result<(), NotmuchError> {
+        let tag = format!("-{tag}");
+        let mut args = vec!["tag", &tag];
+        args.extend(search_term);
+        self.run_notmuch(&args)?;
         Ok(())
     }
 
@@ -598,6 +610,11 @@ impl Notmuch {
 
     #[instrument(skip_all, fields(id=id,part=part))]
     pub fn show_original_part(&self, id: &MessageId, part: usize) -> Result<Vec<u8>, NotmuchError> {
+        let id = if id.starts_with("id:") {
+            id
+        } else {
+            &format!("id:{id}")
+        };
         let res = self.run_notmuch(["show", "--part", &part.to_string(), id])?;
         Ok(res)
     }
@@ -700,6 +717,13 @@ impl Notmuch {
         cmd.args(args);
         info!("{:?}", &cmd);
         let out = cmd.output()?;
+        if !out.stderr.is_empty() {
+            warn!(
+                "{:?}: STDERR:\n{}",
+                &cmd,
+                String::from_utf8_lossy(&out.stderr)
+            );
+        }
         Ok(out.stdout)
     }
 }

procmail2notmuch/Cargo.toml

@@ -11,8 +11,10 @@ version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.69"
+anyhow = "1.0.98"
 clap = { version = "4.5.37", features = ["derive", "env"] }
+letterbox-notmuch = { version = "0.17.9", registry = "xinu" }
+letterbox-shared = { version = "0.17.9", registry = "xinu" }
 serde = { version = "1.0.219", features = ["derive"] }
 sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio"] }
 tokio = { version = "1.44.2", features = ["rt", "macros", "rt-multi-thread"] }

procmail2notmuch/src/main.rs

@@ -1,171 +1,15 @@
-use std::{collections::HashMap, convert::Infallible, io::Write, str::FromStr};
+use std::{collections::HashMap, io::Write};
 
 use clap::{Parser, Subcommand};
-use serde::{Deserialize, Serialize};
+use letterbox_shared::{cleanup_match, Match, MatchType, Rule};
 use sqlx::{types::Json, PgPool};
 
-#[derive(
-    Copy, Clone, Debug, Default, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize,
-)]
-enum MatchType {
-    From,
-    Sender,
-    To,
-    Cc,
-    Subject,
-    ListId,
-    DeliveredTo,
-    XForwardedTo,
-    ReplyTo,
-    XOriginalTo,
-    XSpam,
-    Body,
-    #[default]
-    Unknown,
-}
-#[derive(Debug, Default, Serialize, Deserialize)]
-struct Match {
-    match_type: MatchType,
-    needle: String,
-}
-
-#[derive(Debug, Default, Serialize, Deserialize)]
-struct Rule {
-    stop_on_match: bool,
-    matches: Vec<Match>,
-    tag: Option<String>,
-}
-
-fn unescape(s: &str) -> String {
-    s.replace('\\', "")
-}
-
-fn cleanup_match(prefix: &str, s: &str) -> String {
-    unescape(&s[prefix.len()..]).replace(".*", "")
-}
-
-mod matches {
-    // From https://linux.die.net/man/5/procmailrc
-    // If the regular expression contains '^TO_' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^-a-zA-Z0-9_.])?)'
-    // If the regular expression contains '^TO' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^a-zA-Z])?)', which should catch all destination specifications containing a specific word.
-
-    pub const TO: &'static str = "TO";
-    pub const CC: &'static str = "Cc";
-    pub const TOCC: &'static str = "(TO|Cc)";
-    pub const FROM: &'static str = "From";
-    pub const SENDER: &'static str = "Sender";
-    pub const SUBJECT: &'static str = "Subject";
-    pub const DELIVERED_TO: &'static str = "Delivered-To";
-    pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
-    pub const REPLY_TO: &'static str = "Reply-To";
-    pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
-    pub const LIST_ID: &'static str = "List-ID";
-    pub const X_SPAM: &'static str = "X-Spam";
-    pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
-}
-
-impl FromStr for Match {
-    type Err = Infallible;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        // Examples:
-        //   "* 1^0 ^TOsonyrewards.com@xinu.tv"
-        //   "* ^TOsonyrewards.com@xinu.tv"
-        let mut it = s.split_whitespace().skip(1);
-        let mut needle = it.next().unwrap();
-        if needle == "1^0" {
-            needle = it.next().unwrap();
-        }
-        let mut needle = vec![needle];
-        needle.extend(it);
-        let needle = needle.join(" ");
-        let first = needle.chars().nth(0).unwrap_or(' ');
-        use matches::*;
-        if first == '^' {
-            let needle = &needle[1..];
-            if needle.starts_with(TO) {
-                return Ok(Match {
-                    match_type: MatchType::To,
-                    needle: cleanup_match(TO, needle),
-                });
-            } else if needle.starts_with(FROM) {
-                return Ok(Match {
-                    match_type: MatchType::From,
-                    needle: cleanup_match(FROM, needle),
-                });
-            } else if needle.starts_with(CC) {
-                return Ok(Match {
-                    match_type: MatchType::Cc,
-                    needle: cleanup_match(CC, needle),
-                });
-            } else if needle.starts_with(TOCC) {
-                return Ok(Match {
-                    match_type: MatchType::To,
-                    needle: cleanup_match(TOCC, needle),
-                });
-            } else if needle.starts_with(SENDER) {
-                return Ok(Match {
-                    match_type: MatchType::Sender,
-                    needle: cleanup_match(SENDER, needle),
-                });
-            } else if needle.starts_with(SUBJECT) {
-                return Ok(Match {
-                    match_type: MatchType::Subject,
-                    needle: cleanup_match(SUBJECT, needle),
-                });
-            } else if needle.starts_with(X_ORIGINAL_TO) {
-                return Ok(Match {
-                    match_type: MatchType::XOriginalTo,
-                    needle: cleanup_match(X_ORIGINAL_TO, needle),
-                });
-            } else if needle.starts_with(LIST_ID) {
-                return Ok(Match {
-                    match_type: MatchType::ListId,
-                    needle: cleanup_match(LIST_ID, needle),
-                });
-            } else if needle.starts_with(REPLY_TO) {
-                return Ok(Match {
-                    match_type: MatchType::ReplyTo,
-                    needle: cleanup_match(REPLY_TO, needle),
-                });
-            } else if needle.starts_with(X_SPAM_FLAG) {
-                return Ok(Match {
-                    match_type: MatchType::XSpam,
-                    needle: '*'.to_string(),
-                });
-            } else if needle.starts_with(X_SPAM) {
-                return Ok(Match {
-                    match_type: MatchType::XSpam,
-                    needle: '*'.to_string(),
-                });
-            } else if needle.starts_with(DELIVERED_TO) {
-                return Ok(Match {
-                    match_type: MatchType::DeliveredTo,
-                    needle: cleanup_match(DELIVERED_TO, needle),
-                });
-            } else if needle.starts_with(X_FORWARDED_TO) {
-                return Ok(Match {
-                    match_type: MatchType::XForwardedTo,
-                    needle: cleanup_match(X_FORWARDED_TO, needle),
-                });
-            } else {
-                unreachable!("needle: '{needle}'")
-            }
-        } else {
-            return Ok(Match {
-                match_type: MatchType::Body,
-                needle: cleanup_match("", &needle),
-            });
-        }
-    }
-}
-
 #[derive(Debug, Subcommand)]
 enum Mode {
     Debug,
     Notmuchrc,
     LoadSql {
-        #[arg(short, long, default_value = env!("DATABASE_URL"))]
+        #[arg(short, long)]
         dsn: String,
     },
 }
@@ -204,6 +48,9 @@ async fn main() -> anyhow::Result<()> {
         match first {
             ':' => {
                 // start of rule
+
+                // If carbon-copy flag present, don't stop on match
+                cur_rule.stop_on_match = !l.contains('c');
             }
             '*' => {
                 // add to current rule
@@ -212,13 +59,13 @@ async fn main() -> anyhow::Result<()> {
             }
             '.' => {
                 // delivery to folder
-                cur_rule.tag = Some(cleanup_match(
+                cur_rule.tag = cleanup_match(
                     "",
                     &l.replace('.', "/")
                         .replace(' ', "")
                         .trim_matches('/')
                         .to_string(),
-                ));
+                );
                 rules.push(cur_rule);
                 cur_rule = Rule::default();
             }
@@ -226,7 +73,7 @@ async fn main() -> anyhow::Result<()> {
             '|' => cur_rule = Rule::default(), // external command
             '$' => {
                 // TODO(wathiede): tag messages with no other tag as 'inbox'
-                cur_rule.tag = Some(cleanup_match("", "inbox"));
+                cur_rule.tag = cleanup_match("", "inbox");
                 rules.push(cur_rule);
                 cur_rule = Rule::default();
             } // variable, should only be $DEFAULT in my config
@@ -262,42 +109,41 @@ fn notmuch_from_rules<W: Write>(mut w: W, rules: &[Rule]) -> anyhow::Result<()>
     let mut lines = Vec::new();
     for r in rules {
         for m in &r.matches {
-            if let Some(t) = &r.tag {
+            let t = &r.tag;
-                if let MatchType::Unknown = m.match_type {
+            if let MatchType::Unknown = m.match_type {
-                    eprintln!("rule has unknown match {:?}", r);
+                eprintln!("rule has unknown match {:?}", r);
-                    continue;
+                continue;
-                }
-
-                let rule = match m.match_type {
-                    MatchType::From => "from:",
-                    // TODO(wathiede): something more specific?
-                    MatchType::Sender => "from:",
-                    MatchType::To => "to:",
-                    MatchType::Cc => "to:",
-                    MatchType::Subject => "subject:",
-                    MatchType::ListId => "List-ID:",
-                    MatchType::Body => "",
-                    // TODO(wathiede): these will probably require adding fields to notmuch
-                    // index. Handle them later.
-                    MatchType::DeliveredTo
-                    | MatchType::XForwardedTo
-                    | MatchType::ReplyTo
-                    | MatchType::XOriginalTo
-                    | MatchType::XSpam => continue,
-                    MatchType::Unknown => unreachable!(),
-                };
-                // Preserve unread status if run with --remove-all
-                lines.push(format!(
-                    r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
-                    t, rule, m.needle
-                ));
-                lines.push(format!(
-                    // TODO(wathiede): this assumes `notmuch new` is configured to add
-                    // `tag:unprocessed` to all new mail.
-                    r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
-                    t, rule, m.needle
-                ));
             }
+
+            let rule = match m.match_type {
+                MatchType::From => "from:",
+                // TODO(wathiede): something more specific?
+                MatchType::Sender => "from:",
+                MatchType::To => "to:",
+                MatchType::Cc => "to:",
+                MatchType::Subject => "subject:",
+                MatchType::ListId => "List-ID:",
+                MatchType::Body => "",
+                // TODO(wathiede): these will probably require adding fields to notmuch
+                // index. Handle them later.
+                MatchType::DeliveredTo
+                | MatchType::XForwardedTo
+                | MatchType::ReplyTo
+                | MatchType::XOriginalTo
+                | MatchType::XSpam => continue,
+                MatchType::Unknown => unreachable!(),
+            };
+            // Preserve unread status if run with --remove-all
+            lines.push(format!(
+                r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
+                t, rule, m.needle
+            ));
+            lines.push(format!(
+                // TODO(wathiede): this assumes `notmuch new` is configured to add
+                // `tag:unprocessed` to all new mail.
+                r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
+                t, rule, m.needle
+            ));
         }
     }
     lines.sort();

server/.sqlx/query-126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nSELECT\n  url\nFROM email_photo ep\nJOIN email_address ea\nON ep.id = ea.email_photo_id\nWHERE\n  address = $1\n    ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "url",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "126e16a4675e8d79f330b235f9e1b8614ab1e1526e4e69691c5ebc70d54a42ef"
-}

server/Cargo.toml

@@ -12,48 +12,52 @@ version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ammonia = "4.0.0"
+ammonia = "4.1.0"
-anyhow = "1.0.79"
+anyhow = "1.0.98"
+askama = { version = "0.14.0", features = ["derive"] }
 async-graphql = { version = "7", features = ["log"] }
-async-graphql-axum = "7.0.15"
+async-graphql-axum = "7.0.16"
-async-trait = "0.1.81"
+async-trait = "0.1.88"
 axum = { version = "0.8.3", features = ["ws"] }
 axum-macros = "0.5.0"
-build-info = "0.0.40"
+build-info = "0.0.41"
 cacher = { version = "0.2.0", registry = "xinu" }
-chrono = "0.4.39"
+chrono = "0.4.40"
-clap = { version = "4.5.36", features = ["derive"] }
+clap = { version = "4.5.37", features = ["derive"] }
-css-inline = "0.14.0"
+css-inline = "0.17.0"
+flate2 = "1.1.2"
 futures = "0.3.31"
 headers = "0.4.0"
 html-escape = "0.2.13"
-letterbox-notmuch = { version = "0.15.11", path = "../notmuch", registry = "xinu" }
+letterbox-notmuch = { path = "../notmuch", version = "0.17.34", registry = "xinu" }
-letterbox-shared = { version = "0.15.11", path = "../shared", registry = "xinu" }
+letterbox-shared = { path = "../shared", version = "0.17.34", registry = "xinu" }
 linkify = "0.10.0"
-log = "0.4.17"
+lol_html = "2.3.0"
-lol_html = "2.0.0"
+mailparse = "0.16.1"
-mailparse = "0.16.0"
 maplit = "1.0.2"
 memmap = "0.7.0"
+quick-xml = { version = "0.38.1", features = ["serialize"] }
 regex = "1.11.1"
-reqwest = { version = "0.12.7", features = ["blocking"] }
+reqwest = { version = "0.12.15", features = ["blocking"] }
-scraper = "0.23.0"
+scraper = "0.23.1"
-serde = { version = "1.0.147", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
-serde_json = "1.0.87"
+serde_json = "1.0.140"
-sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "time"] }
+sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio", "time"] }
-tantivy = { version = "0.24.0", optional = true }
+tantivy = { version = "0.24.1", optional = true }
-thiserror = "2.0.0"
+thiserror = "2.0.12"
-tokio = "1.26.0"
+tokio = "1.44.2"
 tower-http = { version = "0.6.2", features = ["trace"] }
 tracing = "0.1.41"
-url = "2.5.2"
+url = "2.5.4"
 urlencoding = "2.1.3"
 #xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" }
 #xtracing = { path = "../../xtracing" }
-xtracing = { version = "0.3.0", registry = "xinu" }
+xtracing = { version = "0.3.2", registry = "xinu" }
+zip = "4.3.0"
+
 
 [build-dependencies]
-build-info-build = "0.0.40"
+build-info-build = "0.0.41"
 
 [features]
 #default = [ "tantivy" ]

server/build.rs

@@ -2,4 +2,5 @@ fn main() {
     // Calling `build_info_build::build_script` collects all data and makes it available to `build_info::build_info!`
     // and `build_info::format!` in the main program.
     build_info_build::build_script();
+    println!("cargo:rerun-if-changed=templates");
 }

server/migrations/20250623193718_disable-feed.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE feed DROP COLUMN IF EXISTS disabled;

server/migrations/20250623193718_disable-feed.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE feed ADD disabled boolean;

server/sql/label_unprocessed.sql

@@ -0,0 +1 @@
+SELECT rule as "rule: Json<Rule>" FROM email_rule ORDER BY sort_order

server/sql/photo_url_for_email_address.sql

@@ -0,0 +1 @@
+SELECT url FROM email_photo ep JOIN email_address ea ON ep.id = ea.email_photo_id WHERE address = $1

server/src/bin/letterbox-server.rs

@@ -21,7 +21,7 @@ use letterbox_notmuch::Notmuch;
 use letterbox_server::tantivy::TantivyConnection;
 use letterbox_server::{
     graphql::{compute_catchup_ids, Attachment, MutationRoot, QueryRoot, SubscriptionRoot},
-    nm::{attachment_bytes, cid_attachment_bytes},
+    nm::{attachment_bytes, cid_attachment_bytes, label_unprocessed},
     ws::ConnectionTracker,
 };
 use letterbox_shared::WebsocketMessage;
@@ -29,9 +29,9 @@ use serde::Deserialize;
 use sqlx::postgres::PgPool;
 use tokio::{net::TcpListener, sync::Mutex};
 use tower_http::trace::{DefaultMakeSpan, TraceLayer};
-use tracing::{info, warn};
+use tracing::{error, info};
 
-// Make our own error that wraps `anyhow::Error`.
+// Make our own error that wraps `ServerError`.
 struct AppError(letterbox_server::ServerError);
 
 // Tell axum how to convert `AppError` into a response.
@@ -148,18 +148,9 @@ async fn view_original(
     extract::Path(id): extract::Path<String>,
 ) -> Result<impl IntoResponse, AppError> {
     info!("view_original {id}");
-    let mid = if id.starts_with("id:") {
+    let bytes = nm.show_original(&id)?;
-        id.to_string()
+    let s = String::from_utf8_lossy(&bytes).to_string();
-    } else {
+    Ok(s.into_response())
-        format!("id:{}", id)
-    };
-    let files = nm.files(&mid)?;
-    let Some(path) = files.first() else {
-        warn!("failed to find files for message {mid}");
-        return Ok((StatusCode::NOT_FOUND, mid).into_response());
-    };
-    let str = std::fs::read_to_string(&path)?;
-    Ok(str.into_response())
 }
 
 async fn graphiql() -> impl IntoResponse {
@@ -185,11 +176,15 @@ async fn start_ws(
 #[derive(Debug, Deserialize)]
 struct NotificationParams {
     delay_ms: Option<u64>,
+    num_unprocessed: Option<usize>,
 }
 
 async fn send_refresh_websocket_handler(
     State(AppState {
-        connection_tracker, ..
+        nm,
+        pool,
+        connection_tracker,
+        ..
     }): State<AppState>,
     params: Query<NotificationParams>,
 ) -> impl IntoResponse {
@@ -199,12 +194,27 @@ async fn send_refresh_websocket_handler(
         info!("sleeping {delay:?}");
         tokio::time::sleep(delay).await;
     }
+    let limit = match params.num_unprocessed {
+        Some(0) => None,
+        Some(limit) => Some(limit),
+        None => Some(10),
+    };
+
+    let mut ids = None;
+    match label_unprocessed(&nm, &pool, false, limit, "tag:unprocessed").await {
+        Ok(i) => ids = Some(i),
+        Err(err) => error!("Failed to label_unprocessed: {err:?}"),
+    };
     connection_tracker
         .lock()
         .await
         .send_message_all(WebsocketMessage::RefreshMessages)
         .await;
-    "refresh triggered"
+    if let Some(ids) = ids {
+        format!("{ids:?}")
+    } else {
+        "refresh triggered".to_string()
+    }
 }
 
 async fn watch_new(
@@ -213,18 +223,33 @@ async fn watch_new(
     conn_tracker: Arc<Mutex<ConnectionTracker>>,
     poll_time: Duration,
 ) -> Result<(), async_graphql::Error> {
-    let mut old_ids = Vec::new();
+    async fn watch_new_iteration(
-    loop {
+        nm: &Notmuch,
+        pool: &PgPool,
+        conn_tracker: Arc<Mutex<ConnectionTracker>>,
+        old_ids: &[String],
+    ) -> Result<Vec<String>, async_graphql::Error> {
         let ids = compute_catchup_ids(&nm, &pool, "is:unread").await?;
+        info!("old_ids: {} ids: {}", old_ids.len(), ids.len());
         if old_ids != ids {
-            info!("old_ids: {old_ids:?}\n     ids: {ids:?}");
+            label_unprocessed(&nm, &pool, false, Some(100), "tag:unprocessed").await?;
             conn_tracker
                 .lock()
                 .await
                 .send_message_all(WebsocketMessage::RefreshMessages)
                 .await
         }
-        old_ids = ids;
+        Ok(ids)
+    }
+    let mut old_ids = Vec::new();
+    loop {
+        old_ids = match watch_new_iteration(&nm, &pool, conn_tracker.clone(), &old_ids).await {
+            Ok(old_ids) => old_ids,
+            Err(err) => {
+                error!("watch_new_iteration failed: {err:?}");
+                continue;
+            }
+        };
         tokio::time::sleep(poll_time).await;
     }
 }
@@ -232,6 +257,7 @@ async fn watch_new(
 #[derive(Clone)]
 struct AppState {
     nm: Notmuch,
+    pool: PgPool,
     connection_tracker: Arc<Mutex<ConnectionTracker>>,
 }
 
@@ -272,7 +298,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let connection_tracker = Arc::new(Mutex::new(ConnectionTracker::default()));
     let ct = Arc::clone(&connection_tracker);
     let poll_time = Duration::from_secs(60);
-    let _h = tokio::spawn(watch_new(nm.clone(), pool, ct, poll_time));
+    let _h = tokio::spawn(watch_new(nm.clone(), pool.clone(), ct, poll_time));
 
     let api_routes = Router::new()
         .route(
@@ -297,6 +323,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .nest("/notification", notification_routes)
         .with_state(AppState {
             nm,
+            pool,
             connection_tracker,
         })
         .layer(

server/src/bin/test-labeling.rs

@@ -0,0 +1,39 @@
+use std::error::Error;
+
+use clap::Parser;
+use letterbox_notmuch::Notmuch;
+use letterbox_server::nm::label_unprocessed;
+use sqlx::postgres::PgPool;
+use tracing::info;
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    #[arg(short, long)]
+    newsreader_database_url: String,
+    #[arg(short, long, default_value = "10")]
+    /// Set to 0 to process all matches
+    messages_to_process: usize,
+    #[arg(short, long, default_value = "false")]
+    execute: bool,
+    /// Process messages matching this notmuch query
+    #[arg(short, long, default_value = "tag:unprocessed")]
+    query: String,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
+    let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
+    build_info::build_info!(fn bi);
+    info!("Build Info: {}", letterbox_shared::build_version(bi));
+    let pool = PgPool::connect(&cli.newsreader_database_url).await?;
+    let nm = Notmuch::default();
+    let limit = if cli.messages_to_process > 0 {
+        Some(cli.messages_to_process)
+    } else {
+        None
+    };
+    label_unprocessed(&nm, &pool, !cli.execute, limit, &cli.query).await?;
+    Ok(())
+}

server/src/error.rs

@@ -39,4 +39,10 @@ pub enum ServerError {
     QueryParseError(#[from] QueryParserError),
     #[error("impossible: {0}")]
     InfaillibleError(#[from] Infallible),
+    #[error("askama error: {0}")]
+    AskamaError(#[from] askama::Error),
+    #[error("xml error: {0}")]
+    XmlError(#[from] quick_xml::Error),
+    #[error("xml encoding error: {0}")]
+    XmlEncodingError(#[from] quick_xml::encoding::EncodingError),
 }

server/src/graphql.rs

@@ -9,15 +9,14 @@ use async_graphql::{
 use cacher::FilesystemCacher;
 use futures::stream;
 use letterbox_notmuch::Notmuch;
-use log::info;
 use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPool;
 use tokio::join;
-use tracing::instrument;
+use tracing::{info, instrument};
 
 #[cfg(feature = "tantivy")]
 use crate::tantivy::TantivyConnection;
-use crate::{newsreader, nm, Query};
+use crate::{newsreader, nm, nm::label_unprocessed, Query};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -238,6 +237,22 @@ impl Body {
             content_tree: "".to_string(),
         })
     }
+
+    pub fn to_html(&self) -> Option<String> {
+        match self {
+            Body::Html(h) => Some(h.html.clone()),
+            Body::PlainText(p) => Some(format!("<pre>{}</pre>", html_escape::encode_text(&p.text))),
+            Body::UnhandledContentType(u) => Some(format!("<pre>{}</pre>", html_escape::encode_text(&u.text))),
+        }
+    }
+
+    pub fn to_html_content_tree(&self) -> Option<String> {
+        match self {
+            Body::Html(h) => Some(h.content_tree.clone()),
+            Body::PlainText(p) => Some(p.content_tree.clone()),
+            Body::UnhandledContentType(u) => Some(u.content_tree.clone()),
+        }
+    }
 }
 
 #[derive(Debug, SimpleObject)]
@@ -629,6 +644,10 @@ impl MutationRoot {
         let pool = ctx.data_unchecked::<PgPool>();
         info!("{}", String::from_utf8_lossy(&nm.new()?));
         newsreader::refresh(pool, cacher).await?;
+
+        // Process email labels
+        label_unprocessed(&nm, &pool, false, Some(10), "tag:unprocessed").await?;
+
         #[cfg(feature = "tantivy")]
         {
             let tantivy = ctx.data_unchecked::<TantivyConnection>();

server/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod config;
+pub mod email_extract;
 pub mod error;
 pub mod graphql;
 pub mod newsreader;
@@ -21,7 +22,6 @@ use cacher::{Cacher, FilesystemCacher};
 use css_inline::{CSSInliner, InlineError, InlineOptions};
 pub use error::ServerError;
 use linkify::{LinkFinder, LinkKind};
-use log::{debug, error, info, warn};
 use lol_html::{
     element, errors::RewritingError, html_content::ContentType, rewrite_str, text,
     RewriteStrSettings,
@@ -32,6 +32,7 @@ use reqwest::StatusCode;
 use scraper::{Html, Selector};
 use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
+use tracing::{debug, error, info, warn};
 use url::Url;
 
 use crate::{

server/src/newsreader.rs

@@ -3,11 +3,10 @@ use std::collections::HashMap;
 use cacher::FilesystemCacher;
 use futures::{stream::FuturesUnordered, StreamExt};
 use letterbox_shared::compute_color;
-use log::{error, info};
 use maplit::hashmap;
 use scraper::Selector;
 use sqlx::postgres::PgPool;
-use tracing::instrument;
+use tracing::{error, info, instrument};
 use url::Url;
 
 use crate::{
@@ -353,6 +352,9 @@ fn slurp_contents_selectors() -> HashMap<String, Vec<Selector>> {
         "natwelch.com".to_string() => vec![
             Selector::parse("article div.prose").unwrap(),
         ],
+        "seiya.me".to_string() => vec![
+            Selector::parse("header + div").unwrap(),
+        ],
         "rustacean-station.org".to_string() => vec![
             Selector::parse("article").unwrap(),
         ],

server/src/templates.rs

@@ -0,0 +1,7 @@
+use askama::Template;
+
+#[derive(Template)]
+#[template(path = "dmarc_report.html")]
+pub struct DmarcReportTemplate<'a> {
+    pub feedback: &'a crate::nm::Feedback,
+}

server/templates/dmarc_report.html

@@ -0,0 +1,99 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>DMARC Report</title>
+</head>
+<body>
+    {% if report.report_metadata.is_some() %}
+        {% let meta = report.report_metadata.as_ref().unwrap() %}
+        <b>Reporter:</b> {{ meta.org_name }}<br>
+        <b>Contact:</b> {{ meta.email }}<br>
+        <b>Report ID:</b> {{ meta.report_id }}<br>
+        {% if meta.date_range.is_some() %}
+            {% let dr = meta.date_range.as_ref().unwrap() %}
+            <b>Date range:</b>
+            {{ dr.begin }}
+            to
+            {{ dr.end }}
+            <br>
+        {% endif %}
+    {% endif %}
+
+    {% if report.policy_published.is_some() %}
+        {% let pol = report.policy_published.as_ref().unwrap() %}
+        <b>Policy Published:</b>
+        <ul>
+            <li>Domain: {{ pol.domain }}</li>
+            <li>ADKIM: {{ pol.adkim }}</li>
+            <li>ASPF: {{ pol.aspf }}</li>
+            <li>Policy: {{ pol.p }}</li>
+            <li>Subdomain Policy: {{ pol.sp }}</li>
+            <li>Percent: {{ pol.pct }}</li>
+        </ul>
+    {% endif %}
+
+    {% if report.record.is_some() %}
+        <b>Records:</b>
+        <table style="border-collapse:collapse;width:100%;font-size:0.95em;">
+            <thead>
+                <tr style="background:#f0f0f0;">
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Source IP</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Count</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Header From</th>
+                    {% if report.has_envelope_to %}
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Envelope To</th>
+                    {% endif %}
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Disposition</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">DKIM</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">SPF</th>
+                    <th style="border:1px solid #bbb;padding:4px 8px;">Auth Results</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for rec in report.record.as_ref().unwrap() %}
+                    <tr>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.source_ip }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.count }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.header_from }}</td>
+                        {% if report.has_envelope_to %}
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.envelope_to }}</td>
+                        {% endif %}
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.disposition }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.dkim }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">{{ rec.spf }}</td>
+                        <td style="border:1px solid #bbb;padding:4px 8px;">
+                            {% if rec.auth_results.is_some() %}
+                                {% let auth = rec.auth_results.as_ref().unwrap() %}
+                                {% for dkimres in auth.dkim %}
+                                        <span style="white-space:nowrap;">
+                                            DKIM: domain=<b>{{ dkimres.domain }}</b>
+                                            selector=<b>{{ dkimres.selector }}</b>
+                                            result=<b>{{ dkimres.result }}</b>
+                                        </span><br>
+                                    {% endfor %}
+
+                                {% for spfres in auth.spf %}
+                                        <span style="white-space:nowrap;">
+                                            SPF: domain=<b>{{ spfres.domain }}</b>
+                                            scope=<b>{{ spfres.scope }}</b>
+                                            result=<b>{{ spfres.result }}</b>
+                                        </span><br>
+                                    {% endfor %}
+
+                            {% for reason in rec.reason %}
+                                <span style="white-space:nowrap;">Reason: {{ reason }}</span><br>
+                            {% endfor %}
+
+                            {% endif %}
+                        </td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    {% endif %}
+
+    {% if report.report_metadata.is_none() && report.policy_published.is_none() && report.record.is_none() %}
+        <p>No DMARC summary found.</p>
+    {% endif %}
+</body>
+</html>

server/templates/tls_report.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>TLS Report</title>
+</head>
+<body>
+    <h3>TLS Report Summary:</h3>
+    <p>Organization: {{ report.organization_name }}</p>
+    <p>Date Range: {{ report.date_range.start_datetime }} to {{ report.date_range.end_datetime }}</p>
+    <p>Contact: {{ report.contact_info }}</p>
+    <p>Report ID: {{ report.report_id }}</p>
+
+    <h4>Policies:</h4>
+    {% for policy in report.policies %}
+        <h5>Policy Domain: {{ policy.policy.policy_domain }}</h5>
+        <ul>
+            <li>Policy Type: {{ policy.policy.policy_type }}</li>
+            <li>Policy String: {{ policy.policy.policy_string | join(", ") }}</li>
+            <li>Successful Sessions: {{ policy.summary.total_successful_session_count }}</li>
+            <li>Failed Sessions: {{ policy.summary.total_failure_session_count }}</li>
+        </ul>
+
+        <ul>
+                {% for mx_host in policy.policy.mx_host %}
+                    <li>Hostname: {{ mx_host.hostname }}, Failures: {{ mx_host.failure_count }}, Result: {{ mx_host.result_type }}</li>
+                {% endfor %}
+            </ul>
+
+        <ul>
+                {% for detail in policy.failure_details %}
+                    <li>Result: {{ detail.result_type }}, Sending IP: {{ detail.sending_mta_ip }}, Failed Sessions: {{ detail.failed_session_count }}
+                        {% if detail.failure_reason_code != "" %}
+                            (Reason: {{ detail.failure_reason_code }})
+                        {% endif %}
+                    </li>
+                    (Receiving IP: {{ detail.receiving_ip }})
+                    (Receiving MX: {{ detail.receiving_mx_hostname }})
+                    (Additional Info: {{ detail.additional_info }})
+                {% endfor %}
+            </ul>
+    {% endfor %}
+</body>
+</html>

server/testdata/dmarc-example-no-envelope-to.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+    <version>1.0</version>
+    <report_metadata>
+        <org_name>google.com</org_name>
+        <email>noreply-dmarc-support@google.com</email>
+        <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
+        <report_id>5142106658860834914</report_id>
+        <date_range>
+            <begin>1755302400</begin>
+            <end>1755388799</end>
+        </date_range>
+    </report_metadata>
+    <policy_published>
+        <domain>xinu.tv</domain>
+        <adkim>s</adkim>
+        <aspf>s</aspf>
+        <p>quarantine</p>
+        <sp>reject</sp>
+        <pct>100</pct>
+        <np>reject</np>
+    </policy_published>
+    <record>
+        <row>
+            <source_ip>74.207.253.222</source_ip>
+            <count>1</count>
+            <policy_evaluated>
+                <disposition>none</disposition>
+                <dkim>pass</dkim>
+                <spf>pass</spf>
+            </policy_evaluated>
+        </row>
+        <identifiers>
+            <header_from>xinu.tv</header_from>
+        </identifiers>
+        <auth_results>
+            <dkim>
+                <domain>xinu.tv</domain>
+                <result>pass</result>
+                <selector>mail</selector>
+            </dkim>
+            <spf>
+                <domain>xinu.tv</domain>
+                <result>pass</result>
+            </spf>
+        </auth_results>
+    </record>
+</feedback>

server/testdata/dmarc-example.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<feedback xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <version>1.0</version>
+    <report_metadata>
+        <org_name>Outlook.com</org_name>
+        <email>dmarcreport@microsoft.com</email>
+        <report_id>e6c5a2ce6e074d7d8cd041a0d6f32a3d</report_id>
+        <date_range>
+            <begin>1755302400</begin>
+            <end>1755388800</end>
+        </date_range>
+    </report_metadata>
+    <policy_published>
+        <domain>xinu.tv</domain>
+        <adkim>s</adkim>
+        <aspf>s</aspf>
+        <p>quarantine</p>
+        <sp>reject</sp>
+        <pct>100</pct>
+        <fo>1</fo>
+    </policy_published>
+    <record>
+        <row>
+            <source_ip>74.207.253.222</source_ip>
+            <count>1</count>
+            <policy_evaluated>
+                <disposition>none</disposition>
+                <dkim>pass</dkim>
+                <spf>pass</spf>
+            </policy_evaluated>
+        </row>
+        <identifiers>
+            <envelope_to>msn.com</envelope_to>
+            <envelope_from>xinu.tv</envelope_from>
+            <header_from>xinu.tv</header_from>
+        </identifiers>
+        <auth_results>
+            <dkim>
+                <domain>xinu.tv</domain>
+                <selector>mail</selector>
+                <result>pass</result>
+            </dkim>
+            <spf>
+                <domain>xinu.tv</domain>
+                <scope>mfrom</scope>
+                <result>pass</result>
+            </spf>
+        </auth_results>
+    </record>
+    <record>
+        <row>
+            <source_ip>74.207.253.222</source_ip>
+            <count>1</count>
+            <policy_evaluated>
+                <disposition>none</disposition>
+                <dkim>pass</dkim>
+                <spf>pass</spf>
+            </policy_evaluated>
+        </row>
+        <identifiers>
+            <envelope_to>hotmail.com</envelope_to>
+            <envelope_from>xinu.tv</envelope_from>
+            <header_from>xinu.tv</header_from>
+        </identifiers>
+        <auth_results>
+            <dkim>
+                <domain>xinu.tv</domain>
+                <selector>mail</selector>
+                <result>pass</result>
+            </dkim>
+            <spf>
+                <domain>xinu.tv</domain>
+                <scope>mfrom</scope>
+                <result>pass</result>
+            </spf>
+        </auth_results>
+    </record>
+</feedback>

shared/Cargo.toml

@@ -11,7 +11,10 @@ version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-build-info = "0.0.40"
+build-info = "0.0.41"
-letterbox-notmuch = { version = "0.15.11", path = "../notmuch", registry = "xinu" }
+letterbox-notmuch = { path = "../notmuch", version = "0.17.34", registry = "xinu" }
-serde = { version = "1.0.147", features = ["derive"] }
+regex = "1.11.1"
+serde = { version = "1.0.219", features = ["derive"] }
+sqlx = "0.8.5"
 strum_macros = "0.27.1"
+tracing = "0.1.41"

shared/src/lib.rs

@@ -1,8 +1,14 @@
-use std::hash::{DefaultHasher, Hash, Hasher};
+use std::{
+    convert::Infallible,
+    hash::{DefaultHasher, Hash, Hasher},
+    str::FromStr,
+};
 
 use build_info::{BuildInfo, VersionControl};
 use letterbox_notmuch::SearchSummary;
+use regex::{RegexBuilder, RegexSetBuilder};
 use serde::{Deserialize, Serialize};
+use tracing::debug;
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SearchResult {
@@ -65,3 +71,198 @@ pub fn compute_color(data: &str) -> String {
     data.hash(&mut hasher);
     format!("#{:06x}", hasher.finish() % (1 << 24))
 }
+
+#[derive(
+    Copy, Clone, Debug, Default, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize,
+)]
+pub enum MatchType {
+    From,
+    Sender,
+    To,
+    Cc,
+    Subject,
+    ListId,
+    DeliveredTo,
+    XForwardedTo,
+    ReplyTo,
+    XOriginalTo,
+    XSpam,
+    Body,
+    #[default]
+    Unknown,
+}
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct Match {
+    pub match_type: MatchType,
+    pub needle: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct Rule {
+    pub stop_on_match: bool,
+    pub matches: Vec<Match>,
+    pub tag: String,
+}
+impl Rule {
+    pub fn is_match(&self, header_key: &str, header_value: &str) -> bool {
+        let pats: Vec<_> = self
+            .matches
+            .iter()
+            .filter_map(|m| match m.match_type {
+                MatchType::To => Some("^(to|cc|bcc|x-original-to)$"),
+                MatchType::From => Some("^from$"),
+                MatchType::Sender => Some("^sender$"),
+                MatchType::Subject => Some("^subject$"),
+                MatchType::ListId => Some("^list-id$"),
+                MatchType::XOriginalTo => Some("^x-original-to$"),
+                MatchType::ReplyTo => Some("^reply-to$"),
+                MatchType::XSpam => Some("^x-spam$"),
+                MatchType::Body => None,
+                c => panic!("TODO handle '{c:?}' match type"),
+            })
+            .collect();
+
+        let set = RegexSetBuilder::new(&pats)
+            .case_insensitive(true)
+            .build()
+            .expect("failed to compile regex for matches");
+        let matches: Vec<_> = set.matches(header_key).into_iter().collect();
+        if !matches.is_empty() {
+            //info!("matched key '{header_key}' '{header_value}'");
+            for m_idx in matches {
+                let needle = regex::escape(&self.matches[m_idx].needle);
+                let pat = RegexBuilder::new(&needle)
+                    .case_insensitive(true)
+                    .build()
+                    .expect("failed to compile regex for needle");
+                if pat.is_match(header_value) {
+                    debug!("{header_key} matched {header_value} against {needle}");
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}
+
+mod matches {
+    // From https://linux.die.net/man/5/procmailrc
+    // If the regular expression contains '^TO_' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^-a-zA-Z0-9_.])?)'
+    // If the regular expression contains '^TO' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^a-zA-Z])?)', which should catch all destination specifications containing a specific word.
+
+    pub const TO: &'static str = "TO";
+    pub const CC: &'static str = "Cc";
+    pub const TOCC: &'static str = "(TO|Cc)";
+    pub const FROM: &'static str = "From";
+    pub const SENDER: &'static str = "Sender";
+    pub const SUBJECT: &'static str = "Subject";
+    pub const DELIVERED_TO: &'static str = "Delivered-To";
+    pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
+    pub const REPLY_TO: &'static str = "Reply-To";
+    pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
+    pub const LIST_ID: &'static str = "List-ID";
+    pub const X_SPAM: &'static str = "X-Spam";
+    pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
+}
+
+impl FromStr for Match {
+    type Err = Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        // Examples:
+        //   "* 1^0 ^TOsonyrewards.com@xinu.tv"
+        //   "* ^TOsonyrewards.com@xinu.tv"
+        let mut it = s.split_whitespace().skip(1);
+        let mut needle = it.next().unwrap();
+        if needle == "1^0" {
+            needle = it.next().unwrap();
+        }
+        let mut needle = vec![needle];
+        needle.extend(it);
+        let needle = needle.join(" ");
+        let first = needle.chars().nth(0).unwrap_or(' ');
+        use matches::*;
+        if first == '^' {
+            let needle = &needle[1..];
+            if needle.starts_with(TO) {
+                return Ok(Match {
+                    match_type: MatchType::To,
+                    needle: cleanup_match(TO, needle),
+                });
+            } else if needle.starts_with(FROM) {
+                return Ok(Match {
+                    match_type: MatchType::From,
+                    needle: cleanup_match(FROM, needle),
+                });
+            } else if needle.starts_with(CC) {
+                return Ok(Match {
+                    match_type: MatchType::Cc,
+                    needle: cleanup_match(CC, needle),
+                });
+            } else if needle.starts_with(TOCC) {
+                return Ok(Match {
+                    match_type: MatchType::To,
+                    needle: cleanup_match(TOCC, needle),
+                });
+            } else if needle.starts_with(SENDER) {
+                return Ok(Match {
+                    match_type: MatchType::Sender,
+                    needle: cleanup_match(SENDER, needle),
+                });
+            } else if needle.starts_with(SUBJECT) {
+                return Ok(Match {
+                    match_type: MatchType::Subject,
+                    needle: cleanup_match(SUBJECT, needle),
+                });
+            } else if needle.starts_with(X_ORIGINAL_TO) {
+                return Ok(Match {
+                    match_type: MatchType::XOriginalTo,
+                    needle: cleanup_match(X_ORIGINAL_TO, needle),
+                });
+            } else if needle.starts_with(LIST_ID) {
+                return Ok(Match {
+                    match_type: MatchType::ListId,
+                    needle: cleanup_match(LIST_ID, needle),
+                });
+            } else if needle.starts_with(REPLY_TO) {
+                return Ok(Match {
+                    match_type: MatchType::ReplyTo,
+                    needle: cleanup_match(REPLY_TO, needle),
+                });
+            } else if needle.starts_with(X_SPAM_FLAG) {
+                return Ok(Match {
+                    match_type: MatchType::XSpam,
+                    needle: '*'.to_string(),
+                });
+            } else if needle.starts_with(X_SPAM) {
+                return Ok(Match {
+                    match_type: MatchType::XSpam,
+                    needle: '*'.to_string(),
+                });
+            } else if needle.starts_with(DELIVERED_TO) {
+                return Ok(Match {
+                    match_type: MatchType::DeliveredTo,
+                    needle: cleanup_match(DELIVERED_TO, needle),
+                });
+            } else if needle.starts_with(X_FORWARDED_TO) {
+                return Ok(Match {
+                    match_type: MatchType::XForwardedTo,
+                    needle: cleanup_match(X_FORWARDED_TO, needle),
+                });
+            } else {
+                unreachable!("needle: '{needle}'")
+            }
+        } else {
+            return Ok(Match {
+                match_type: MatchType::Body,
+                needle: cleanup_match("", &needle),
+            });
+        }
+    }
+}
+fn unescape(s: &str) -> String {
+    s.replace('\\', "")
+}
+pub fn cleanup_match(prefix: &str, s: &str) -> String {
+    unescape(&s[prefix.len()..]).replace(".*", "")
+}

web/Cargo.toml

@@ -9,33 +9,32 @@ repository.workspace = true
 version.workspace = true
 
 [build-dependencies]
-build-info-build = "0.0.40"
+build-info-build = "0.0.41"
 
 [dev-dependencies]
-wasm-bindgen-test = "0.3.33"
+wasm-bindgen-test = "0.3.50"
 
 [dependencies]
 console_error_panic_hook = "0.1.7"
-log = "0.4.17"
+log = "0.4.27"
 seed = { version = "0.10.0", features = ["routing"] }
 #seed = "0.9.2"
-console_log = { version = "0.1.0", registry = "xinu" }
+console_log = { version = "0.1.4", registry = "xinu" }
-serde = { version = "1.0.147", features = ["derive"] }
+serde = { version = "1.0.219", features = ["derive"] }
 itertools = "0.14.0"
-serde_json = { version = "1.0.93", features = ["unbounded_depth"] }
+serde_json = { version = "1.0.140", features = ["unbounded_depth"] }
-chrono = "0.4.31"
+chrono = "0.4.40"
 graphql_client = "0.14.0"
-thiserror = "2.0.0"
+thiserror = "2.0.12"
 gloo-net = { version = "0.6.0", features = ["json", "serde_json"] }
 human_format = "1.1.0"
-build-info = "0.0.40"
+build-info = "0.0.41"
 wasm-bindgen = "=0.2.100"
-uuid = { version = "1.13.1", features = [
+uuid = { version = "1.16.0", features = [
   "js",
 ] } # direct dep to set js feature, prevents Rng issues
-letterbox-shared = { version = "0.15.11", path = "../shared", registry = "xinu" }
+letterbox-shared = { path = "../shared/", version = "0.17.34", registry = "xinu" }
-letterbox-notmuch = { version = "0.15.11", path = "../notmuch", registry = "xinu" }
+seed_hooks = { version = "0.4.1", registry = "xinu" }
-seed_hooks = { version = "0.4.0", registry = "xinu" }
 strum_macros = "0.27.1"
 gloo-console = "0.3.0"
 [target.'cfg(target_arch = "wasm32")'.dependencies]
@@ -45,7 +44,7 @@ wasm-sockets = "1.0.0"
 wasm-opt = ['-Os']
 
 [dependencies.web-sys]
-version = "0.3.58"
+version = "0.3.77"
 features = [
   "Clipboard",
   "DomRect",

web/index.html

@@ -16,10 +16,11 @@
   <link data-trunk rel="css" href="static/vars.css" />
   <link data-trunk rel="tailwind-css" href="./src/tailwind.css" />
   <link data-trunk rel="css" href="static/overrides.css" />
+  <link data-trunk rel="css" href="static/email-specific.css" />
 </head>
 
 <body>
   <section id="app"></section>
 </body>
 
-</html>
+</html>

web/src/main.rs

@@ -2,8 +2,6 @@
 //  - it's useful when you want to check your code with `cargo make verify`
 // but some rules are too "annoying" or are not applicable for your case.)
 #![allow(clippy::wildcard_imports)]
-// Until https://github.com/rust-lang/rust/issues/138762 is addressed in dependencies
-#![allow(wasm_c_abi)]
 
 use log::Level;
 use seed::App;

web/src/state.rs

@@ -72,10 +72,6 @@ fn on_url_changed(old: &Url, mut new: Url) -> Msg {
     if did_change {
         messages.push(Msg::ScrollToTop)
     }
-    info!(
-        "url changed\nold '{old}'\nnew '{new}', history {}",
-        history().length().unwrap_or(0)
-    );
     let hpp = new.remaining_hash_path_parts();
     let msg = match hpp.as_slice() {
         ["t", tid] => Msg::ShowThreadRequest {
@@ -553,7 +549,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             });
         }
         Msg::ScrollToTop => {
-            info!("scrolling to the top");
             web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
         }
         Msg::WindowScrolled => {
@@ -619,6 +614,36 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             orders.send_msg(Msg::CatchupRequest { query });
         }
         Msg::CatchupKeepUnread => {
+            if let Some(thread_id) = current_thread_id(&model.context) {
+                if let Context::ThreadResult {
+                    thread:
+                        ShowThreadQueryThread::EmailThread(ShowThreadQueryThreadOnEmailThread {
+                            messages,
+                            ..
+                        }),
+                    ..
+                } = &model.context
+                {
+                    //orders.send_msg(Msg::SetUnread(thread_id, false));
+                    let unread_messages: Vec<_> = messages
+                        .iter()
+                        .filter(|msg| msg.tags.iter().any(|t| t == "unread"))
+                        .map(|msg| &msg.id)
+                        .collect();
+                    if unread_messages.is_empty() {
+                        // All messages are read, so mark them all unread
+                        orders.send_msg(Msg::SetUnread(thread_id, true));
+                    } else {
+                        // Do nothing if there are some messages unread
+                    }
+                } else {
+                    // News post, not email, just mark unread
+                    orders.send_msg(Msg::SetUnread(thread_id, true));
+                };
+            } else {
+                // This shouldn't happen
+                warn!("no current thread_id");
+            }
             orders.send_msg(Msg::CatchupNext);
         }
         Msg::CatchupMarkAsRead => {

web/src/view/mod.rs

@@ -341,9 +341,10 @@ fn search_results(
         let caught_up = query.contains("is:unread");
         let read_emoji = ["👻", "👽", "👾", "🤖", "💀"];
         let no_results_emoji = ["🙈", "👀", "🤦", "🤷", "🙅", "🛟", "🍩", "🌑", "💿", "🔍"];
+        // Randomly choose emoji based on what 10-second window we're currently in
         let now = seed::window()
             .performance()
-            .map(|p| p.now() as usize)
+            .map(|p| p.now() as usize / 10_000)
             .unwrap_or(0);
         let (emoji, text) = if caught_up {
             let idx = now % read_emoji.len();
@@ -1024,7 +1025,7 @@ fn message_render(msg: &ShowThreadQueryThreadOnEmailThreadMessages, open: bool)
         ],
         IF!(open =>
         div![
-            C!["content", "bg-white", "text-black", "p-4", "min-w-full", "w-0","overflow-x-auto", from],
+            C!["content", "bg-white", "text-black", "p-4", "min-w-full", "w-0","overflow-x-auto", from.map(|f|format!("from-{f}"))],
             match &msg.body {
                 ShowThreadQueryThreadOnEmailThreadMessagesBody::UnhandledContentType(
                     ShowThreadQueryThreadOnEmailThreadMessagesBodyOnUnhandledContentType { contents ,content_tree},

web/src/websocket.rs

@@ -1,7 +1,7 @@
 use std::{collections::VecDeque, rc::Rc};
 
 use letterbox_shared::WebsocketMessage;
-use log::{error, info};
+use log::{debug, error};
 use seed::prelude::*;
 use serde::{Deserialize, Serialize};
 #[cfg(not(target_arch = "wasm32"))]
@@ -63,13 +63,6 @@ use wasm_sockets::{ConnectionStatus, EventClient, Message, WebSocketError};
 use wasm_sockets::{ConnectionStatus, EventClient, Message, WebSocketError};
 use web_sys::CloseEvent;
 
-/// Message from the server to the client.
-#[derive(Serialize, Deserialize)]
-pub struct ServerMessage {
-    pub id: usize,
-    pub text: String,
-}
-
 /// Message from the client to the server.
 #[derive(Serialize, Deserialize)]
 pub struct ClientMessage {
@@ -122,13 +115,13 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
     match msg {
         Msg::WebSocketOpened => {
             model.web_socket_reconnector = None;
-            info!("WebSocket connection is open now");
+            debug!("WebSocket connection is open now");
         }
         Msg::TextMessageReceived(msg) => {
             model.updates.push_back(msg);
         }
         Msg::WebSocketClosed(close_event) => {
-            info!(
+            debug!(
                 r#"==================
 WebSocket connection was closed:
 Clean: {0}
@@ -148,7 +141,7 @@ Reason: {2}
             }
         }
         Msg::WebSocketFailed => {
-            info!("WebSocket failed");
+            debug!("WebSocket failed");
             if model.web_socket_reconnector.is_none() {
                 model.web_socket_reconnector = Some(
                     orders.stream_with_handle(streams::backoff(None, Msg::ReconnectWebSocket)),
@@ -156,7 +149,7 @@ Reason: {2}
             }
         }
         Msg::ReconnectWebSocket(retries) => {
-            info!("Reconnect attempt: {}", retries);
+            debug!("Reconnect attempt: {}", retries);
             model.web_socket = create_websocket(&model.ws_url, orders).unwrap();
         }
         Msg::SendMessage(msg) => {
@@ -177,16 +170,16 @@ fn create_websocket(url: &str, orders: &impl Orders<Msg>) -> Result<EventClient,
 
     let send = msg_sender.clone();
     client.set_on_connection(Some(Box::new(move |client: &EventClient| {
-        info!("{:#?}", client.status);
+        debug!("{:#?}", client.status);
         let msg = match *client.status.borrow() {
             ConnectionStatus::Connecting => {
-                info!("Connecting...");
+                debug!("Connecting...");
                 None
             }
             ConnectionStatus::Connected => Some(Msg::WebSocketOpened),
             ConnectionStatus::Error => Some(Msg::WebSocketFailed),
             ConnectionStatus::Disconnected => {
-                info!("Disconnected");
+                debug!("Disconnected");
                 None
             }
         };
@@ -195,7 +188,7 @@ fn create_websocket(url: &str, orders: &impl Orders<Msg>) -> Result<EventClient,
 
     let send = msg_sender.clone();
     client.set_on_close(Some(Box::new(move |ev| {
-        info!("WS: Connection closed");
+        debug!("WS: Connection closed");
         send(Some(Msg::WebSocketClosed(ev)));
     })));
 

web/static/email-specific.css

@@ -0,0 +1,11 @@
+.mail-thread .content.from-noreply-news-bloomberg-com a {
+  background-color: initial !important;
+}
+
+.mail-thread .content.from-noreply-news-bloomberg-com h2 {
+  margin: 0 !important;
+  padding: 0 !important;
+}
+.mail-thread .content.from-dmarcreport-microsoft-com div {
+  font-family:  ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji" !important;
+}

web/static/overrides.css

@@ -57,15 +57,6 @@ html {
   margin-left: 2em;
 }
 
-.mail-thread .content .noreply-news-bloomberg-com a {
-  background-color: initial !important;
-}
-
-.mail-thread .content .noreply-news-bloomberg-com h2 {
-  margin: 0 !important;
-  padding: 0 !important;
-}
-
 /* Hackaday figures have unreadable black on dark grey */
 .news-post figcaption.wp-caption-text {
   background-color: initial !important;
@@ -76,6 +67,11 @@ html {
   display: none !important;
 }
 
+.news-post.site-seiya-me figure>pre,
+.news-post.site-seiya-me figure>pre>code {
+  background-color: black !important;
+}
+
 .news-post.site-slashdot .story-byline {
   display: block !important;
   height: initial !important;