Compare commits

...

27 Commits

Author SHA1 Message Date
1f79b43a85 chore: Release
Some checks failed
Continuous integration / Check (push) Successful in 37s
Continuous integration / Test Suite (push) Successful in 44s
Continuous integration / Trunk (push) Failing after 36s
Continuous integration / Rustfmt (push) Successful in 38s
Continuous integration / build (push) Successful in 51s
Continuous integration / Disallow unused dependencies (push) Successful in 1m57s
2025-04-21 22:01:49 -07:00
904619bccd chore: Release 2025-04-21 22:01:41 -07:00
14104f6469 Remove non hermetic default flage values
Some checks failed
Continuous integration / Test Suite (push) Successful in 57s
Continuous integration / Trunk (push) Failing after 38s
Continuous integration / Check (push) Successful in 2m1s
Continuous integration / Rustfmt (push) Successful in 32s
Continuous integration / Disallow unused dependencies (push) Successful in 1m14s
Continuous integration / build (push) Successful in 1m37s
2025-04-21 21:59:22 -07:00
dccfb6f71f chore: Release
Some checks failed
Continuous integration / Check (push) Failing after 36s
Continuous integration / Test Suite (push) Failing after 43s
Continuous integration / Trunk (push) Failing after 36s
Continuous integration / Rustfmt (push) Successful in 37s
Continuous integration / build (push) Failing after 51s
Continuous integration / Disallow unused dependencies (push) Failing after 1m58s
2025-04-21 21:20:51 -07:00
547266a705 Fix imports for letterbox-* packages 2025-04-21 21:20:31 -07:00
273562b58c chore: Release 2025-04-21 21:16:43 -07:00
dc39eed1a7 cargo sqlx prepare 2025-04-21 21:16:42 -07:00
9178badfd0 Add mail tagging support 2025-04-21 21:15:55 -07:00
38e75ec251 web: make random emoji selection more deterministic 2025-04-21 10:12:12 -07:00
c1496bf87b server: doc cleanup 2025-04-20 10:48:59 -07:00
4da888b240 Move id format check from server into notmuch 2025-04-20 10:47:40 -07:00
c703be2ca5 server: more robust view original serving 2025-04-20 10:01:22 -07:00
5cec8add5e chore: Release
Some checks failed
Continuous integration / Check (push) Successful in 42s
Continuous integration / Trunk (push) Failing after 38s
Continuous integration / Test Suite (push) Successful in 1m20s
Continuous integration / Rustfmt (push) Successful in 32s
Continuous integration / build (push) Successful in 1m20s
Continuous integration / Disallow unused dependencies (push) Successful in 1m0s
2025-04-20 09:46:49 -07:00
0225dbde3a procmail2notmuch: don't run migration code, leave it to server 2025-04-20 09:46:27 -07:00
f84b8fa6c2 chore: Release 2025-04-20 09:38:35 -07:00
979cbcd23e procmail2notmuch: inlude early exit option 2025-04-20 09:37:51 -07:00
b3070e1919 web: use random emoji when search results empty, handle search vs catchup 2025-04-20 09:37:12 -07:00
e5fdde8f30 web: add graphic when search results are empty 2025-04-20 09:07:43 -07:00
7de36bbc3d procmail2notmuch: add sql rule loader 2025-04-20 08:40:06 -07:00
1c4f27902e server: add todo 2025-04-20 08:39:47 -07:00
7ee86f0d2f chore: Release
Some checks failed
Continuous integration / Check (push) Successful in 36s
Continuous integration / Test Suite (push) Successful in 41s
Continuous integration / Trunk (push) Failing after 35s
Continuous integration / Rustfmt (push) Successful in 38s
Continuous integration / build (push) Successful in 47s
Continuous integration / Disallow unused dependencies (push) Successful in 1m57s
2025-04-19 13:19:14 -07:00
a0b06fd5ef chore: Release 2025-04-19 13:17:01 -07:00
630bb20b35 procmail2notmuch: add debug vs notmuchrc modes 2025-04-19 13:16:47 -07:00
17ea2a35cb web: tweak style and behavior of view original link 2025-04-19 13:11:57 -07:00
7d9376d607 Add view original functionality 2025-04-19 12:33:11 -07:00
122e949072 chore: Release
Some checks failed
Continuous integration / Test Suite (push) Successful in 41s
Continuous integration / Check (push) Successful in 1m33s
Continuous integration / Trunk (push) Failing after 35s
Continuous integration / Rustfmt (push) Successful in 56s
Continuous integration / build (push) Successful in 48s
Continuous integration / Disallow unused dependencies (push) Successful in 3m14s
2025-04-16 08:48:35 -07:00
9a69b4c51e web: scroll to top on pagination 2025-04-16 08:47:45 -07:00
18 changed files with 770 additions and 351 deletions

66
Cargo.lock generated
View File

@@ -863,9 +863,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.36"
version = "4.5.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071"
dependencies = [
"clap_builder",
"clap_derive",
@@ -873,9 +873,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.36"
version = "4.5.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2"
dependencies = [
"anstream",
"anstyle",
@@ -2995,7 +2995,21 @@ dependencies = [
[[package]]
name = "letterbox-notmuch"
version = "0.15.6"
version = "0.16.0"
source = "sparse+https://git.z.xinu.tv/api/packages/wathiede/cargo/"
checksum = "f7cdd2798042f4cc63342d798f450e7152231f4f592b3142cd63a0a9f4b879d8"
dependencies = [
"log",
"mailparse",
"serde",
"serde_json",
"thiserror 2.0.12",
"tracing",
]
[[package]]
name = "letterbox-notmuch"
version = "0.17.2"
dependencies = [
"itertools",
"log",
@@ -3010,14 +3024,20 @@ dependencies = [
[[package]]
name = "letterbox-procmail2notmuch"
version = "0.15.6"
version = "0.17.2"
dependencies = [
"anyhow",
"clap",
"letterbox-notmuch 0.16.0",
"letterbox-shared 0.16.0",
"serde",
"sqlx",
"tokio 1.44.2",
]
[[package]]
name = "letterbox-server"
version = "0.15.6"
version = "0.17.2"
dependencies = [
"ammonia",
"anyhow",
@@ -3035,8 +3055,8 @@ dependencies = [
"futures 0.3.31",
"headers",
"html-escape",
"letterbox-notmuch",
"letterbox-shared",
"letterbox-notmuch 0.16.0",
"letterbox-shared 0.16.0",
"linkify",
"log",
"lol_html",
@@ -3061,17 +3081,35 @@ dependencies = [
[[package]]
name = "letterbox-shared"
version = "0.15.6"
version = "0.16.0"
source = "sparse+https://git.z.xinu.tv/api/packages/wathiede/cargo/"
checksum = "18fcc018014a200754ea7524f41fc4e5e14f5edc4cb0ca5d7afbaa476cb0d297"
dependencies = [
"build-info",
"letterbox-notmuch",
"letterbox-notmuch 0.16.0",
"regex",
"serde",
"sqlx",
"strum_macros 0.27.1",
"tracing",
]
[[package]]
name = "letterbox-shared"
version = "0.17.2"
dependencies = [
"build-info",
"letterbox-notmuch 0.16.0",
"regex",
"serde",
"sqlx",
"strum_macros 0.27.1",
"tracing",
]
[[package]]
name = "letterbox-web"
version = "0.15.6"
version = "0.17.2"
dependencies = [
"build-info",
"build-info-build",
@@ -3083,8 +3121,8 @@ dependencies = [
"graphql_client",
"human_format",
"itertools",
"letterbox-notmuch",
"letterbox-shared",
"letterbox-notmuch 0.16.0",
"letterbox-shared 0.16.0",
"log",
"seed",
"seed_hooks",

View File

@@ -8,7 +8,7 @@ authors = ["Bill Thiede <git@xinu.tv>"]
edition = "2021"
license = "UNLICENSED"
publish = ["xinu"]
version = "0.15.6"
version = "0.17.2"
repository = "https://git.z.xinu.tv/wathiede/letterbox"
[profile.dev]

View File

@@ -598,6 +598,11 @@ impl Notmuch {
#[instrument(skip_all, fields(id=id,part=part))]
pub fn show_original_part(&self, id: &MessageId, part: usize) -> Result<Vec<u8>, NotmuchError> {
let id = if id.starts_with("id:") {
id
} else {
&format!("id:{id}")
};
let res = self.run_notmuch(["show", "--part", &part.to_string(), id])?;
Ok(res)
}

View File

@@ -12,3 +12,9 @@ version.workspace = true
[dependencies]
anyhow = "1.0.69"
clap = { version = "4.5.37", features = ["derive", "env"] }
letterbox-notmuch = { version = "0.16.0", registry = "xinu" }
letterbox-shared = { version = "0.16.0", registry = "xinu" }
serde = { version = "1.0.219", features = ["derive"] }
sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio"] }
tokio = { version = "1.44.2", features = ["rt", "macros", "rt-multi-thread"] }

View File

@@ -1,210 +1,36 @@
use std::{convert::Infallible, io::Write, str::FromStr};
use std::{collections::HashMap, io::Write};
#[derive(Debug, Default)]
enum MatchType {
From,
Sender,
To,
Cc,
Subject,
List,
DeliveredTo,
XForwardedTo,
ReplyTo,
XOriginalTo,
XSpam,
Body,
#[default]
Unknown,
}
#[derive(Debug, Default)]
struct Match {
match_type: MatchType,
needle: String,
use clap::{Parser, Subcommand};
use letterbox_shared::{cleanup_match, Match, MatchType, Rule};
use sqlx::{types::Json, PgPool};
#[derive(Debug, Subcommand)]
enum Mode {
Debug,
Notmuchrc,
LoadSql {
#[arg(short, long)]
dsn: String,
},
}
#[derive(Debug, Default)]
struct Rule {
matches: Vec<Match>,
tags: Vec<String>,
/// Simple program to greet a person
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
struct Args {
#[arg(short, long, default_value = "/home/wathiede/dotfiles/procmailrc")]
input: String,
#[command(subcommand)]
mode: Mode,
}
fn unescape(s: &str) -> String {
s.replace('\\', "")
}
fn cleanup_match(prefix: &str, s: &str) -> String {
unescape(&s[prefix.len()..]).replace(".*", "")
}
mod matches {
pub const TO: &'static str = "TO";
pub const CC: &'static str = "Cc";
pub const TOCC: &'static str = "(TO|Cc)";
pub const FROM: &'static str = "From";
pub const SENDER: &'static str = "Sender";
pub const SUBJECT: &'static str = "Subject";
pub const DELIVERED_TO: &'static str = "Delivered-To";
pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
pub const REPLY_TO: &'static str = "Reply-To";
pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
pub const LIST_ID: &'static str = "List-ID";
pub const X_SPAM: &'static str = "X-Spam";
pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
}
impl FromStr for Match {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
// Examples:
// "* 1^0 ^TOsonyrewards.com@xinu.tv"
// "* ^TOsonyrewards.com@xinu.tv"
let mut it = s.split_whitespace().skip(1);
let mut needle = it.next().unwrap();
if needle == "1^0" {
needle = it.next().unwrap();
}
let mut needle = vec![needle];
needle.extend(it);
let needle = needle.join(" ");
let first = needle.chars().nth(0).unwrap_or(' ');
use matches::*;
if first == '^' {
let needle = &needle[1..];
if needle.starts_with(TO) {
return Ok(Match {
match_type: MatchType::To,
needle: cleanup_match(TO, needle),
});
} else if needle.starts_with(FROM) {
return Ok(Match {
match_type: MatchType::From,
needle: cleanup_match(FROM, needle),
});
} else if needle.starts_with(CC) {
return Ok(Match {
match_type: MatchType::Cc,
needle: cleanup_match(CC, needle),
});
} else if needle.starts_with(TOCC) {
return Ok(Match {
match_type: MatchType::To,
needle: cleanup_match(TOCC, needle),
});
} else if needle.starts_with(SENDER) {
return Ok(Match {
match_type: MatchType::Sender,
needle: cleanup_match(SENDER, needle),
});
} else if needle.starts_with(SUBJECT) {
return Ok(Match {
match_type: MatchType::Subject,
needle: cleanup_match(SUBJECT, needle),
});
} else if needle.starts_with(X_ORIGINAL_TO) {
return Ok(Match {
match_type: MatchType::XOriginalTo,
needle: cleanup_match(X_ORIGINAL_TO, needle),
});
} else if needle.starts_with(LIST_ID) {
return Ok(Match {
match_type: MatchType::List,
needle: cleanup_match(LIST_ID, needle),
});
} else if needle.starts_with(REPLY_TO) {
return Ok(Match {
match_type: MatchType::ReplyTo,
needle: cleanup_match(REPLY_TO, needle),
});
} else if needle.starts_with(X_SPAM_FLAG) {
return Ok(Match {
match_type: MatchType::XSpam,
needle: '*'.to_string(),
});
} else if needle.starts_with(X_SPAM) {
return Ok(Match {
match_type: MatchType::XSpam,
needle: '*'.to_string(),
});
} else if needle.starts_with(DELIVERED_TO) {
return Ok(Match {
match_type: MatchType::DeliveredTo,
needle: cleanup_match(DELIVERED_TO, needle),
});
} else if needle.starts_with(X_FORWARDED_TO) {
return Ok(Match {
match_type: MatchType::XForwardedTo,
needle: cleanup_match(X_FORWARDED_TO, needle),
});
} else {
unreachable!("needle: '{needle}'")
}
} else {
return Ok(Match {
match_type: MatchType::Body,
needle: cleanup_match("", &needle),
});
}
}
}
fn notmuch_from_rules<W: Write>(mut w: W, rules: &[Rule]) -> anyhow::Result<()> {
// TODO(wathiede): if reindexing this many tags is too slow, see if combining rules per tag is
// faster.
let mut lines = Vec::new();
for r in rules {
for m in &r.matches {
for t in &r.tags {
if let MatchType::Unknown = m.match_type {
eprintln!("rule has unknown match {:?}", r);
continue;
}
let rule = match m.match_type {
MatchType::From => "from:",
// TODO(wathiede): something more specific?
MatchType::Sender => "from:",
MatchType::To => "to:",
MatchType::Cc => "to:",
MatchType::Subject => "subject:",
MatchType::List => "List-ID:",
MatchType::Body => "",
// TODO(wathiede): these will probably require adding fields to notmuch
// index. Handle them later.
MatchType::DeliveredTo
| MatchType::XForwardedTo
| MatchType::ReplyTo
| MatchType::XOriginalTo
| MatchType::XSpam => continue,
MatchType::Unknown => unreachable!(),
};
// Preserve unread status if run with --remove-all
lines.push(format!(
r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
t, rule, m.needle
));
lines.push(format!(
// TODO(wathiede): this assumes `notmuch new` is configured to add
// `tag:unprocessed` to all new mail.
r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
t, rule, m.needle
));
}
}
}
lines.sort();
for l in lines {
writeln!(w, "{l}")?;
}
Ok(())
}
fn main() -> anyhow::Result<()> {
let input = "/home/wathiede/dotfiles/procmailrc";
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let args = Args::parse();
let mut rules = Vec::new();
let mut cur_rule = Rule::default();
for l in std::fs::read_to_string(input)?.lines() {
for l in std::fs::read_to_string(args.input)?.lines() {
let l = if let Some(idx) = l.find('#') {
&l[..idx]
} else {
@@ -222,6 +48,9 @@ fn main() -> anyhow::Result<()> {
match first {
':' => {
// start of rule
// If carbon-copy flag present, don't stop on match
cur_rule.stop_on_match = !l.contains('c');
}
'*' => {
// add to current rule
@@ -230,26 +59,119 @@ fn main() -> anyhow::Result<()> {
}
'.' => {
// delivery to folder
cur_rule.tags.push(cleanup_match(
cur_rule.tag = cleanup_match(
"",
&l.replace('.', "/")
.replace(' ', "")
.trim_matches('/')
.to_string(),
));
);
rules.push(cur_rule);
cur_rule = Rule::default();
}
'/' => cur_rule = Rule::default(), // Ex. /dev/null
'|' => cur_rule = Rule::default(), // external command
'$' => {
// TODO(wathiede): tag messages with no other tag as 'inbox'
cur_rule.tags.push(cleanup_match("", "inbox"));
cur_rule.tag = cleanup_match("", "inbox");
rules.push(cur_rule);
cur_rule = Rule::default();
} // variable, should only be $DEFAULT in my config
_ => panic!("Unhandled first character '{}' {}", first, l),
_ => panic!("Unhandled first character '{}'\nLine: {}", first, l),
}
}
notmuch_from_rules(std::io::stdout(), &rules)?;
match args.mode {
Mode::Debug => print_rules(&rules),
Mode::Notmuchrc => notmuch_from_rules(std::io::stdout(), &rules)?,
Mode::LoadSql { dsn } => load_sql(&dsn, &rules).await?,
}
Ok(())
}
fn print_rules(rules: &[Rule]) {
let mut tally = HashMap::new();
for r in rules {
for m in &r.matches {
*tally.entry(m.match_type).or_insert(0) += 1;
}
}
let mut sorted: Vec<_> = tally.iter().map(|(k, v)| (v, k)).collect();
sorted.sort();
sorted.reverse();
for (v, k) in sorted {
println!("{k:?}: {v}");
}
}
fn notmuch_from_rules<W: Write>(mut w: W, rules: &[Rule]) -> anyhow::Result<()> {
// TODO(wathiede): if reindexing this many tags is too slow, see if combining rules per tag is
// faster.
let mut lines = Vec::new();
for r in rules {
for m in &r.matches {
let t = &r.tag;
if let MatchType::Unknown = m.match_type {
eprintln!("rule has unknown match {:?}", r);
continue;
}
let rule = match m.match_type {
MatchType::From => "from:",
// TODO(wathiede): something more specific?
MatchType::Sender => "from:",
MatchType::To => "to:",
MatchType::Cc => "to:",
MatchType::Subject => "subject:",
MatchType::ListId => "List-ID:",
MatchType::Body => "",
// TODO(wathiede): these will probably require adding fields to notmuch
// index. Handle them later.
MatchType::DeliveredTo
| MatchType::XForwardedTo
| MatchType::ReplyTo
| MatchType::XOriginalTo
| MatchType::XSpam => continue,
MatchType::Unknown => unreachable!(),
};
// Preserve unread status if run with --remove-all
lines.push(format!(
r#"-unprocessed +{} +unread -- is:unread tag:unprocessed {}"{}""#,
t, rule, m.needle
));
lines.push(format!(
// TODO(wathiede): this assumes `notmuch new` is configured to add
// `tag:unprocessed` to all new mail.
r#"-unprocessed +{} -- tag:unprocessed {}"{}""#,
t, rule, m.needle
));
}
}
lines.sort();
for l in lines {
writeln!(w, "{l}")?;
}
Ok(())
}
async fn load_sql(dsn: &str, rules: &[Rule]) -> anyhow::Result<()> {
let pool = PgPool::connect(dsn).await?;
println!("clearing email_rule table");
sqlx::query!("DELETE FROM email_rule")
.execute(&pool)
.await?;
for (order, rule) in rules.iter().enumerate() {
println!("inserting {order}: {rule:?}");
sqlx::query!(
r#"
INSERT INTO email_rule (sort_order, rule)
VALUES ($1, $2)
"#,
order as i32,
Json(rule) as _
)
.execute(&pool)
.await?;
}
Ok(())
}

View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT rule as \"rule: Json<Rule>\"\n FROM email_rule\n ORDER BY sort_order\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "rule: Json<Rule>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "6c5b0a96f45f78795732ea428cc01b4eab28b7150aa37387e7439a6b0b58e88c"
}

View File

@@ -27,8 +27,8 @@ css-inline = "0.14.0"
futures = "0.3.31"
headers = "0.4.0"
html-escape = "0.2.13"
letterbox-notmuch = { version = "0.15.6", path = "../notmuch", registry = "xinu" }
letterbox-shared = { version = "0.15.6", path = "../shared", registry = "xinu" }
letterbox-notmuch = { version = "0.16.0", registry = "xinu" }
letterbox-shared = { version = "0.16.0", registry = "xinu" }
linkify = "0.10.0"
log = "0.4.17"
lol_html = "2.0.0"

View File

@@ -0,0 +1,3 @@
DROP TABLE IF NOT EXISTS email_rule;
-- Add down migration script here

View File

@@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS email_rule (
id integer NOT NULL GENERATED ALWAYS AS IDENTITY,
sort_order integer NOT NULL,
rule jsonb NOT NULL
);

View File

@@ -31,7 +31,7 @@ use tokio::{net::TcpListener, sync::Mutex};
use tower_http::trace::{DefaultMakeSpan, TraceLayer};
use tracing::info;
// Make our own error that wraps `anyhow::Error`.
// Make our own error that wraps `ServerError`.
struct AppError(letterbox_server::ServerError);
// Tell axum how to convert `AppError` into a response.
@@ -142,6 +142,17 @@ async fn view_cid(
Ok(inline_attachment_response(attachment))
}
// TODO make this work with gitea message ids like `wathiede/letterbox/pulls/91@git.z.xinu.tv`
async fn view_original(
State(AppState { nm, .. }): State<AppState>,
extract::Path(id): extract::Path<String>,
) -> Result<impl IntoResponse, AppError> {
info!("view_original {id}");
let bytes = nm.show_original(&id)?;
let s = String::from_utf8_lossy(&bytes).to_string();
Ok(s.into_response())
}
async fn graphiql() -> impl IntoResponse {
response::Html(
GraphiQLSource::build()
@@ -260,6 +271,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
get(download_attachment),
)
.route("/view/attachment/{id}/{idx}/{*rest}", get(view_attachment))
.route("/original/{id}", get(view_original))
.route("/cid/{id}/{cid}", get(view_cid))
.route("/ws", any(start_ws))
.route_service("/graphql/ws", GraphQLSubscription::new(schema.clone()))

View File

@@ -0,0 +1,39 @@
use std::error::Error;
use clap::Parser;
use letterbox_notmuch::Notmuch;
use letterbox_server::nm::label_unprocessed;
use sqlx::postgres::PgPool;
use tracing::info;
#[derive(Parser)]
#[command(version, about, long_about = None)]
struct Cli {
#[arg(short, long)]
newsreader_database_url: String,
#[arg(short, long, default_value = "10")]
/// Set to 0 to process all matches
messages_to_process: usize,
#[arg(short, long, default_value = "false")]
execute: bool,
/// Process messages matching this notmuch query
#[arg(short, long, default_value = "tag:unprocessed")]
query: String,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
let cli = Cli::parse();
let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?;
build_info::build_info!(fn bi);
info!("Build Info: {}", letterbox_shared::build_version(bi));
let pool = PgPool::connect(&cli.newsreader_database_url).await?;
let nm = Notmuch::default();
let limit = if cli.messages_to_process > 0 {
Some(cli.messages_to_process)
} else {
None
};
label_unprocessed(&nm, &pool, !cli.execute, limit, &cli.query).await?;
Ok(())
}

View File

@@ -17,7 +17,7 @@ use tracing::instrument;
#[cfg(feature = "tantivy")]
use crate::tantivy::TantivyConnection;
use crate::{newsreader, nm, Query};
use crate::{newsreader, nm, nm::label_unprocessed, Query};
/// # Number of seconds since the Epoch
pub type UnixTime = isize;
@@ -629,6 +629,10 @@ impl MutationRoot {
let pool = ctx.data_unchecked::<PgPool>();
info!("{}", String::from_utf8_lossy(&nm.new()?));
newsreader::refresh(pool, cacher).await?;
// Process email labels
label_unprocessed(&nm, &pool, false, Some(10), "tag:unprocessed").await?;
#[cfg(feature = "tantivy")]
{
let tantivy = ctx.data_unchecked::<TantivyConnection>();

View File

@@ -1,11 +1,14 @@
use std::{collections::HashMap, fs::File};
use std::{
collections::{HashMap, HashSet},
fs::File,
};
use letterbox_notmuch::Notmuch;
use letterbox_shared::compute_color;
use letterbox_shared::{compute_color, Rule};
use log::{error, info, warn};
use mailparse::{parse_content_type, parse_mail, MailHeader, MailHeaderMap, ParsedMail};
use memmap::MmapOptions;
use sqlx::PgPool;
use sqlx::{types::Json, PgPool};
use tracing::instrument;
use crate::{
@@ -925,3 +928,106 @@ WHERE
.await?;
Ok(row.map(|r| r.url))
}
/*
* grab email_rules table from sql
* For each message with `unprocessed` label
* parse the message
* pass headers for each message through a matcher using email rules
* for each match, add label to message
* if any matches were found, remove unprocessed
* TODO: how to handle inbox label
*/
#[instrument(name="nm::label_unprocessed", skip_all, fields(dryrun=dryrun, limit=?limit, query=%query))]
pub async fn label_unprocessed(
nm: &Notmuch,
pool: &PgPool,
dryrun: bool,
limit: Option<usize>,
query: &str,
) -> Result<(), ServerError> {
use futures::StreamExt;
let ids = nm.message_ids(query)?;
info!(
"Processing {limit:?} of {} messages with '{query}'",
ids.len()
);
let rules: Vec<_> = sqlx::query!(
r#"
SELECT rule as "rule: Json<Rule>"
FROM email_rule
ORDER BY sort_order
"#,
)
.fetch(pool)
.map(|r| r.unwrap().rule.0)
.collect()
.await;
/*
use letterbox_shared::{Match, MatchType};
let rules = vec![Rule {
stop_on_match: false,
matches: vec![Match {
match_type: MatchType::From,
needle: "eftours".to_string(),
}],
tag: "EFTours".to_string(),
}];
*/
info!("Loaded {} rules", rules.len());
let ids = if let Some(limit) = limit {
&ids[..limit]
} else {
&ids[..]
};
for id in ids {
let id = format!("id:{id}");
let files = nm.files(&id)?;
// Only process the first file path is multiple files have the same id
let path = files.iter().next().unwrap();
let file = File::open(&path)?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let m = parse_mail(&mmap)?;
let (matched_rule, add_tags) = find_tags(&rules, &m.headers);
if matched_rule {
if dryrun {
info!(
"\nAdd tags: {add_tags:?}\nTo: {} From: {} Subject: {}\n",
m.headers.get_first_value("to").expect("no from header"),
m.headers.get_first_value("from").expect("no from header"),
m.headers
.get_first_value("subject")
.expect("no subject header")
);
} else {
for t in &add_tags {
nm.tag_add(t, &id)?;
}
if !add_tags.contains("inbox") {
nm.tag_remove("inbox", &id)?;
}
nm.tag_remove("unprocessed", &id)?;
}
}
}
Ok(())
}
fn find_tags<'a, 'b>(rules: &'a [Rule], headers: &'b [MailHeader]) -> (bool, HashSet<&'a str>) {
let mut matched_rule = false;
let mut add_tags = HashSet::new();
for rule in rules {
for hdr in headers {
if rule.is_match(&hdr.get_key(), &hdr.get_value()) {
//info!("Matched {rule:?}");
matched_rule = true;
add_tags.insert(rule.tag.as_str());
if rule.stop_on_match {
return (true, add_tags);
}
}
}
}
return (matched_rule, add_tags);
}

View File

@@ -12,6 +12,9 @@ version.workspace = true
[dependencies]
build-info = "0.0.40"
letterbox-notmuch = { version = "0.15.6", path = "../notmuch", registry = "xinu" }
letterbox-notmuch = { version = "0.16.0", registry = "xinu" }
regex = "1.11.1"
serde = { version = "1.0.147", features = ["derive"] }
sqlx = "0.8.5"
strum_macros = "0.27.1"
tracing = "0.1.41"

View File

@@ -1,8 +1,14 @@
use std::hash::{DefaultHasher, Hash, Hasher};
use std::{
convert::Infallible,
hash::{DefaultHasher, Hash, Hasher},
str::FromStr,
};
use build_info::{BuildInfo, VersionControl};
use letterbox_notmuch::SearchSummary;
use regex::{RegexBuilder, RegexSetBuilder};
use serde::{Deserialize, Serialize};
use tracing::debug;
#[derive(Serialize, Deserialize, Debug)]
pub struct SearchResult {
@@ -20,6 +26,13 @@ pub enum WebsocketMessage {
pub mod urls {
pub const MOUNT_POINT: &'static str = "/api";
pub fn view_original(host: Option<&str>, id: &str) -> String {
if let Some(host) = host {
format!("//{host}/api/original/{id}")
} else {
format!("/api/original/{id}")
}
}
pub fn cid_prefix(host: Option<&str>, cid: &str) -> String {
if let Some(host) = host {
format!("//{host}/api/cid/{cid}/")
@@ -58,3 +71,198 @@ pub fn compute_color(data: &str) -> String {
data.hash(&mut hasher);
format!("#{:06x}", hasher.finish() % (1 << 24))
}
#[derive(
Copy, Clone, Debug, Default, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize,
)]
pub enum MatchType {
From,
Sender,
To,
Cc,
Subject,
ListId,
DeliveredTo,
XForwardedTo,
ReplyTo,
XOriginalTo,
XSpam,
Body,
#[default]
Unknown,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Match {
pub match_type: MatchType,
pub needle: String,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Rule {
pub stop_on_match: bool,
pub matches: Vec<Match>,
pub tag: String,
}
impl Rule {
pub fn is_match(&self, header_key: &str, header_value: &str) -> bool {
let pats: Vec<_> = self
.matches
.iter()
.filter_map(|m| match m.match_type {
MatchType::To => Some("^(to|cc|bcc|x-original-to)$"),
MatchType::From => Some("^from$"),
MatchType::Sender => Some("^sender$"),
MatchType::Subject => Some("^subject$"),
MatchType::ListId => Some("^list-id$"),
MatchType::XOriginalTo => Some("^x-original-to$"),
MatchType::ReplyTo => Some("^reply-to$"),
MatchType::XSpam => Some("^x-spam$"),
MatchType::Body => None,
c => panic!("TODO handle '{c:?}' match type"),
})
.collect();
let set = RegexSetBuilder::new(&pats)
.case_insensitive(true)
.build()
.expect("failed to compile regex for matches");
let matches: Vec<_> = set.matches(header_key).into_iter().collect();
if !matches.is_empty() {
//info!("matched key '{header_key}' '{header_value}'");
for m_idx in matches {
let needle = regex::escape(&self.matches[m_idx].needle);
let pat = RegexBuilder::new(&needle)
.case_insensitive(true)
.build()
.expect("failed to compile regex for needle");
if pat.is_match(header_value) {
debug!("{header_key} matched {header_value} against {needle}");
return true;
}
}
}
false
}
}
mod matches {
// From https://linux.die.net/man/5/procmailrc
// If the regular expression contains '^TO_' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^-a-zA-Z0-9_.])?)'
// If the regular expression contains '^TO' it will be substituted by '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope |Apparently(-Resent)?)-To):(.*[^a-zA-Z])?)', which should catch all destination specifications containing a specific word.
pub const TO: &'static str = "TO";
pub const CC: &'static str = "Cc";
pub const TOCC: &'static str = "(TO|Cc)";
pub const FROM: &'static str = "From";
pub const SENDER: &'static str = "Sender";
pub const SUBJECT: &'static str = "Subject";
pub const DELIVERED_TO: &'static str = "Delivered-To";
pub const X_FORWARDED_TO: &'static str = "X-Forwarded-To";
pub const REPLY_TO: &'static str = "Reply-To";
pub const X_ORIGINAL_TO: &'static str = "X-Original-To";
pub const LIST_ID: &'static str = "List-ID";
pub const X_SPAM: &'static str = "X-Spam";
pub const X_SPAM_FLAG: &'static str = "X-Spam-Flag";
}
impl FromStr for Match {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
// Examples:
// "* 1^0 ^TOsonyrewards.com@xinu.tv"
// "* ^TOsonyrewards.com@xinu.tv"
let mut it = s.split_whitespace().skip(1);
let mut needle = it.next().unwrap();
if needle == "1^0" {
needle = it.next().unwrap();
}
let mut needle = vec![needle];
needle.extend(it);
let needle = needle.join(" ");
let first = needle.chars().nth(0).unwrap_or(' ');
use matches::*;
if first == '^' {
let needle = &needle[1..];
if needle.starts_with(TO) {
return Ok(Match {
match_type: MatchType::To,
needle: cleanup_match(TO, needle),
});
} else if needle.starts_with(FROM) {
return Ok(Match {
match_type: MatchType::From,
needle: cleanup_match(FROM, needle),
});
} else if needle.starts_with(CC) {
return Ok(Match {
match_type: MatchType::Cc,
needle: cleanup_match(CC, needle),
});
} else if needle.starts_with(TOCC) {
return Ok(Match {
match_type: MatchType::To,
needle: cleanup_match(TOCC, needle),
});
} else if needle.starts_with(SENDER) {
return Ok(Match {
match_type: MatchType::Sender,
needle: cleanup_match(SENDER, needle),
});
} else if needle.starts_with(SUBJECT) {
return Ok(Match {
match_type: MatchType::Subject,
needle: cleanup_match(SUBJECT, needle),
});
} else if needle.starts_with(X_ORIGINAL_TO) {
return Ok(Match {
match_type: MatchType::XOriginalTo,
needle: cleanup_match(X_ORIGINAL_TO, needle),
});
} else if needle.starts_with(LIST_ID) {
return Ok(Match {
match_type: MatchType::ListId,
needle: cleanup_match(LIST_ID, needle),
});
} else if needle.starts_with(REPLY_TO) {
return Ok(Match {
match_type: MatchType::ReplyTo,
needle: cleanup_match(REPLY_TO, needle),
});
} else if needle.starts_with(X_SPAM_FLAG) {
return Ok(Match {
match_type: MatchType::XSpam,
needle: '*'.to_string(),
});
} else if needle.starts_with(X_SPAM) {
return Ok(Match {
match_type: MatchType::XSpam,
needle: '*'.to_string(),
});
} else if needle.starts_with(DELIVERED_TO) {
return Ok(Match {
match_type: MatchType::DeliveredTo,
needle: cleanup_match(DELIVERED_TO, needle),
});
} else if needle.starts_with(X_FORWARDED_TO) {
return Ok(Match {
match_type: MatchType::XForwardedTo,
needle: cleanup_match(X_FORWARDED_TO, needle),
});
} else {
unreachable!("needle: '{needle}'")
}
} else {
return Ok(Match {
match_type: MatchType::Body,
needle: cleanup_match("", &needle),
});
}
}
}
fn unescape(s: &str) -> String {
s.replace('\\', "")
}
pub fn cleanup_match(prefix: &str, s: &str) -> String {
unescape(&s[prefix.len()..]).replace(".*", "")
}

View File

@@ -33,8 +33,8 @@ wasm-bindgen = "=0.2.100"
uuid = { version = "1.13.1", features = [
"js",
] } # direct dep to set js feature, prevents Rng issues
letterbox-shared = { version = "0.15.6", path = "../shared", registry = "xinu" }
letterbox-notmuch = { version = "0.15.6", path = "../notmuch", registry = "xinu" }
letterbox-shared = { version = "0.16.0", registry = "xinu" }
letterbox-notmuch = { version = "0.16.0", registry = "xinu" }
seed_hooks = { version = "0.4.0", registry = "xinu" }
strum_macros = "0.27.1"
gloo-console = "0.3.0"
@@ -50,9 +50,10 @@ features = [
"Clipboard",
"DomRect",
"Element",
"History",
"MediaQueryList",
"Navigator",
"Window",
"History",
"Performance",
"ScrollRestoration",
"Window",
]

View File

@@ -263,81 +263,108 @@ fn search_results(
} else {
set_title(query);
}
let rows = results.iter().map(|r| {
let tid = r.thread.clone();
let check_tid = r.thread.clone();
let datetime = human_age(r.timestamp as i64);
let unread_idx = r.tags.iter().position(|e| e == &"unread");
let mut tags = r.tags.clone();
if let Some(idx) = unread_idx {
tags.remove(idx);
};
let is_unread = unread_idx.is_some();
let mut title_break = None;
const TITLE_LENGTH_WRAP_LIMIT: usize = 40;
for w in r.subject.split_whitespace() {
if w.len() > TITLE_LENGTH_WRAP_LIMIT {
title_break = Some(C!["break-all", "text-pretty"]);
let rows: Vec<_> = results
.iter()
.map(|r| {
let tid = r.thread.clone();
let check_tid = r.thread.clone();
let datetime = human_age(r.timestamp as i64);
let unread_idx = r.tags.iter().position(|e| e == &"unread");
let mut tags = r.tags.clone();
if let Some(idx) = unread_idx {
tags.remove(idx);
};
let is_unread = unread_idx.is_some();
let mut title_break = None;
const TITLE_LENGTH_WRAP_LIMIT: usize = 40;
for w in r.subject.split_whitespace() {
if w.len() > TITLE_LENGTH_WRAP_LIMIT {
title_break = Some(C!["break-all", "text-pretty"]);
}
}
}
div![
C![
"flex",
"flex-nowrap",
"w-auto",
"flex-auto",
"py-4",
"border-b",
"border-neutral-800"
],
div![
C!["flex", "items-center", "mr-4"],
input![
C![&tw_classes::CHECKBOX],
attrs! {
At::Type=>"checkbox",
At::Checked=>selected_threads.contains(&tid).as_at_value(),
}
C![
"flex",
"flex-nowrap",
"w-auto",
"flex-auto",
"py-4",
"border-b",
"border-neutral-800"
],
ev(Ev::Input, move |e| {
if let Some(input) = e
.target()
.as_ref()
.expect("failed to get reference to target")
.dyn_ref::<web_sys::HtmlInputElement>()
{
if input.checked() {
Msg::SelectionAddThread(check_tid)
} else {
Msg::SelectionRemoveThread(check_tid)
}
} else {
Msg::Noop
}
}),
],
a![
C!["flex-grow"],
IF!(is_unread => C!["font-bold"]),
attrs! {
At::Href => urls::thread(&tid)
},
div![title_break, &r.subject],
span![C!["text-xs"], pretty_authors(&r.authors)],
div![
C!["flex", "flex-wrap", "justify-between"],
span![tags_chiclet(&tags)],
span![C!["text-sm"], datetime]
C!["flex", "items-center", "mr-4"],
input![
C![&tw_classes::CHECKBOX],
attrs! {
At::Type=>"checkbox",
At::Checked=>selected_threads.contains(&tid).as_at_value(),
}
],
ev(Ev::Input, move |e| {
if let Some(input) = e
.target()
.as_ref()
.expect("failed to get reference to target")
.dyn_ref::<web_sys::HtmlInputElement>()
{
if input.checked() {
Msg::SelectionAddThread(check_tid)
} else {
Msg::SelectionRemoveThread(check_tid)
}
} else {
Msg::Noop
}
}),
],
a![
C!["flex-grow"],
IF!(is_unread => C!["font-bold"]),
attrs! {
At::Href => urls::thread(&tid)
},
div![title_break, &r.subject],
span![C!["text-xs"], pretty_authors(&r.authors)],
div![
C!["flex", "flex-wrap", "justify-between"],
span![tags_chiclet(&tags)],
span![C!["text-sm"], datetime]
]
]
]
]
});
})
.collect();
let show_bulk_edit = !selected_threads.is_empty();
let all_selected = selected_threads.len() == results.len();
let all_selected = (selected_threads.len() == results.len()) && !rows.is_empty();
let content = if rows.is_empty() {
let caught_up = query.contains("is:unread");
let read_emoji = ["👻", "👽", "👾", "🤖", "💀"];
let no_results_emoji = ["🙈", "👀", "🤦", "🤷", "🙅", "🛟", "🍩", "🌑", "💿", "🔍"];
// Randomly choose emoji based on what 10-second window we're currently in
let now = seed::window()
.performance()
.map(|p| p.now() as usize / 10_000)
.unwrap_or(0);
let (emoji, text) = if caught_up {
let idx = now % read_emoji.len();
(read_emoji[idx], "All caught up!")
} else {
let idx = now % no_results_emoji.len();
(no_results_emoji[idx], "No results")
};
div![
C!["text-center"],
h1![C!["text-9xl"], emoji],
p![C!["mt-8", "text-3xl", "font-semibold"], text]
]
} else {
div![rows]
};
div![
C!["flex", "flex-col", "flex-auto", "p-4"],
search_toolbar(count, pager, show_bulk_edit, all_selected),
div![rows],
content,
search_toolbar(count, pager, show_bulk_edit, all_selected),
]
}
@@ -541,13 +568,19 @@ fn search_toolbar(
tw_classes::button(),
IF!(!pager.has_previous_page => attrs!{ At::Disabled=>true }),
"<",
IF!(pager.has_previous_page => ev(Ev::Click, |_| Msg::PreviousPage)),
IF!(pager.has_previous_page => ev(
Ev::Click, |_| Msg::MultiMsg(vec![
Msg::ScrollToTop,
Msg::PreviousPage]))),
],
button![
tw_classes::button(),
IF!(!pager.has_next_page => attrs!{ At::Disabled=>true }),
">",
IF!(pager.has_next_page => ev(Ev::Click, |_| Msg::NextPage))
IF!(pager.has_next_page => ev(
Ev::Click, |_| Msg::MultiMsg(vec![
Msg::ScrollToTop,
Msg::NextPage])))
]
]
]
@@ -688,6 +721,8 @@ fn render_open_header(msg: &ShowThreadQueryThreadOnEmailThreadMessages) -> Node<
.collect();
let show_x_original_to = !*to_xinu.borrow() && msg.x_original_to.is_some();
let show_delivered_to = !*to_xinu.borrow() && !show_x_original_to && msg.delivered_to.is_some();
let host = seed::window().location().host().expect("couldn't get host");
let href = letterbox_shared::urls::view_original(Some(&host), &msg.id);
div![
C!["flex", "p-4", "bg-neutral-800"],
div![avatar],
@@ -769,20 +804,36 @@ fn render_open_header(msg: &ShowThreadQueryThreadOnEmailThreadMessages) -> Node<
C!["text-right"],
msg.timestamp
.map(|ts| div![C!["text-xs", "text-nowrap"], human_age(ts)]),
i![C![
"mx-4",
"read-status",
"far",
if is_unread {
"fa-envelope"
} else {
"fa-envelope-open"
},
]],
ev(Ev::Click, move |e| {
e.stop_propagation();
Msg::SetUnread(id, !is_unread)
}),
div![
C!["p-2"],
i![C![
"mx-4",
"read-status",
"far",
if is_unread {
"fa-envelope"
} else {
"fa-envelope-open"
},
]],
ev(Ev::Click, move |e| {
e.stop_propagation();
Msg::SetUnread(id, !is_unread)
}),
],
div![
C!["text-xs"],
span![a![
attrs! {
At::Href=>href,
At::Target=>"_blank",
},
"View original",
ev(Ev::Click, move |e| {
e.stop_propagation();
})
]]
]
]
]
}
@@ -925,20 +976,23 @@ fn render_closed_header(msg: &ShowThreadQueryThreadOnEmailThreadMessages) -> Nod
C!["text-right"],
msg.timestamp
.map(|ts| div![C!["text-xs", "text-nowrap"], human_age(ts)]),
i![C![
"mx-4",
"read-status",
"far",
if is_unread {
"fa-envelope"
} else {
"fa-envelope-open"
},
]],
ev(Ev::Click, move |e| {
e.stop_propagation();
Msg::SetUnread(id, !is_unread)
}),
div![
C!["p-2"],
i![C![
"mx-4",
"read-status",
"far",
if is_unread {
"fa-envelope"
} else {
"fa-envelope-open"
},
]],
ev(Ev::Click, move |e| {
e.stop_propagation();
Msg::SetUnread(id, !is_unread)
})
],
]
]
}
@@ -971,7 +1025,7 @@ fn message_render(msg: &ShowThreadQueryThreadOnEmailThreadMessages, open: bool)
],
IF!(open =>
div![
C!["bg-white", "text-black", "p-4", "min-w-full", "w-0","overflow-x-auto", from],
C!["content", "bg-white", "text-black", "p-4", "min-w-full", "w-0","overflow-x-auto", from],
match &msg.body {
ShowThreadQueryThreadOnEmailThreadMessagesBody::UnhandledContentType(
ShowThreadQueryThreadOnEmailThreadMessagesBodyOnUnhandledContentType { contents ,content_tree},
@@ -1075,7 +1129,6 @@ fn render_attachements(
]
}
// TODO: add cathup_mode:bool and hide elements when true
#[topo::nested]
fn thread(
thread: &ShowThreadQueryThreadOnEmailThread,
@@ -1166,13 +1219,7 @@ fn thread(
el_ref(content_el),
messages,
IF!(!catchup_mode => click_to_top())
],
/* TODO(wathiede): plumb in orignal id
a![
attrs! {At::Href=>api::original(&thread_node.0.as_ref().expect("message missing").id)},
"Original"
],
*/
]
]
}

View File

@@ -2,23 +2,23 @@ html {
background-color: black;
}
.mail-thread a,
.mail-thread .content a,
.news-post a {
color: var(--color-link) !important;
text-decoration: underline;
}
.mail-thread br,
.mail-thread .content br,
.news-post br {
display: block;
margin-top: 1em;
content: " ";
}
.mail-thread h1,
.mail-thread h2,
.mail-thread h3,
.mail-thread h4,
.mail-thread .content h1,
.mail-thread .content h2,
.mail-thread .content h3,
.mail-thread .content h4,
.news-post h1,
.news-post h2,
.news-post h3,
@@ -27,12 +27,12 @@ html {
margin-bottom: 1em !important;
}
.mail-thread p,
.mail-thread .content p,
.news-post p {
margin-bottom: 1em;
}
.mail-thread pre,
.mail-thread .content pre,
.news-post pre {
font-family: monospace;
background-color: #eee !important;
@@ -40,28 +40,28 @@ html {
white-space: break-spaces;
}
.mail-thread code,
.mail-thread .content code,
.news-post code {
font-family: monospace;
white-space: break-spaces;
background-color: #eee !important;
}
.mail-thread blockquote {
.mail-thread .content blockquote {
padding-left: 1em;
border-left: 2px solid #ddd;
}
.mail-thread ol,
.mail-thread ul {
.mail-thread .content ol,
.mail-thread .content ul {
margin-left: 2em;
}
.mail-thread .noreply-news-bloomberg-com a {
.mail-thread .content .noreply-news-bloomberg-com a {
background-color: initial !important;
}
.mail-thread .noreply-news-bloomberg-com h2 {
.mail-thread .content .noreply-news-bloomberg-com h2 {
margin: 0 !important;
padding: 0 !important;
}