wathiede/letterbox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

server: pretty print raw TLSRPT and DMARC data

Browse Source
This commit is contained in:
Bill Thiede
2025-08-12 16:08:21 -07:00
parent 4d888fbea3
commit 5c42d04598
5 changed files with 672 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/Cargo.toml
View File

@@ -54,6 +54,7 @@ urlencoding = "2.1.3"
#xtracing = { path = "../../xtracing" }
xtracing = { version = "0.3.2", registry = "xinu" }
zip = "4.3.0"
xmlem = "0.1.0"
[build-dependencies]
build-info-build = "0.0.41"

2
server/src/error.rs
View File

@@ -41,4 +41,6 @@ pub enum ServerError {
InfaillibleError(#[from] Infallible),
#[error("askama error: {0}")]
AskamaError(#[from] askama::Error),
#[error("xml error: {0}")]
XmlError(#[from] quick_xml::Error),
}

16
server/src/graphql.rs
View File

@@ -237,6 +237,22 @@ impl Body {
content_tree: "".to_string(),
})
}
pub fn to_html(&self) -> Option<String> {
match self {
Body::Html(h) => Some(h.html.clone()),
Body::PlainText(p) => Some(format!("<pre>{}</pre>", html_escape::encode_text(&p.text))),
Body::UnhandledContentType(u) => Some(format!("<pre>{}</pre>", html_escape::encode_text(&u.text))),
}
}
pub fn to_html_content_tree(&self) -> Option<String> {
match self {
Body::Html(h) => Some(h.content_tree.clone()),
Body::PlainText(p) => Some(p.content_tree.clone()),
Body::UnhandledContentType(u) => Some(u.content_tree.clone()),
}
}
}
#[derive(Debug, SimpleObject)]

270
server/src/nm.rs
View File

@@ -2,6 +2,7 @@ use std::{
collections::{HashMap, HashSet},
fs::File,
io::{Cursor, Read},
str::FromStr,
};
use askama::Template;
@@ -13,6 +14,7 @@ use memmap::MmapOptions;
use quick_xml::de::from_str as xml_from_str;
use sqlx::{types::Json, PgPool};
use tracing::{error, info, info_span, instrument, warn};
use xmlem::{display, Document};
use zip::ZipArchive;
use crate::{
@@ -176,7 +178,7 @@ pub async fn thread(
// display names (that default to the most commonly seen name).
let mut messages = Vec::new();
for (path, id) in std::iter::zip(nm.files(&thread_id)?, nm.message_ids(&thread_id)?) {
let tags = nm.tags_for_query(&format!("id:{id}"))?;
let tags = nm.tags_for_query(&format!("id:{}", id))?;
let file = File::open(&path)?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let m = parse_mail(&mmap)?;
@@ -314,8 +316,105 @@ pub async fn thread(
.collect();
// TODO(wathiede): parse message and fill out attachments
let attachments = extract_attachments(&m, &id)?;
let mut final_body = body;
let mut raw_report_content: Option<String> = None;
// Append TLS report if available
if m.ctype.mimetype.as_str() == MULTIPART_REPORT {
if let Ok(Body::Html(_html_body)) = extract_report(&m, &mut part_addr) {
// Extract raw JSON for pretty printing
if let Some(sp) = m
.subparts
.iter()
.find(|sp| sp.ctype.mimetype.as_str() == "application/tlsrpt+gzip")
{
if let Ok(gz_bytes) = sp.get_body_raw() {
let mut decoder = flate2::read::GzDecoder::new(&gz_bytes[..]);
let mut buffer = Vec::new();
if decoder.read_to_end(&mut buffer).is_ok() {
if let Ok(json_str) = String::from_utf8(buffer) {
raw_report_content = Some(json_str);
}
}
}
}
}
}
// Append DMARC report if available
if m.ctype.mimetype.as_str() == APPLICATION_ZIP {
if let Ok(Body::Html(_html_body)) = extract_zip(&m) {
// Extract raw XML for pretty printing
if let Ok(zip_bytes) = m.get_body_raw() {
if let Ok(mut archive) = ZipArchive::new(Cursor::new(&zip_bytes)) {
for i in 0..archive.len() {
if let Ok(mut file) = archive.by_index(i) {
let name = file.name().to_lowercase();
if name.ends_with(".xml")
&& (name.contains("dmarc") || name.starts_with("google.com!"))
{
let mut xml = String::new();
use std::io::Read;
if file.read_to_string(&mut xml).is_ok() {
raw_report_content = Some(xml);
}
}
}
}
}
}
}
}
if m.ctype.mimetype.as_str() == APPLICATION_GZIP {
if let Ok(Body::Html(_html_body)) = extract_gzip(&m) {
// Extract raw XML for pretty printing
if let Ok(gz_bytes) = m.get_body_raw() {
let mut decoder = flate2::read::GzDecoder::new(&gz_bytes[..]);
let mut xml = String::new();
use std::io::Read;
if decoder.read_to_string(&mut xml).is_ok() {
raw_report_content = Some(xml);
}
}
}
}
if let Some(raw_content) = raw_report_content {
let pretty_printed_content = if m.ctype.mimetype.as_str() == MULTIPART_REPORT {
// Pretty print JSON
if let Ok(parsed_json) = serde_json::from_str::<serde_json::Value>(&raw_content) {
serde_json::to_string_pretty(&parsed_json).unwrap_or(raw_content)
} else {
raw_content
}
} else {
// DMARC reports are XML
// Pretty print XML
let doc_result = Document::from_str(&raw_content);
if let Ok(doc) = doc_result {
doc.to_string_pretty_with_config(&display::Config::default_pretty())
} else {
error!(
"Failed to parse XML for pretty printing: {:?}",
doc_result.unwrap_err()
);
raw_content
}
};
final_body = Body::Html(Html {
html: format!(
"{}\n<pre>{}</pre>",
final_body.to_html().unwrap_or_default(),
html_escape::encode_text(&pretty_printed_content)
),
content_tree: final_body.to_html_content_tree().unwrap_or_default(),
});
}
messages.push(Message {
id: format!("id:{id}"),
id: format!("id:{}", id),
from,
to,
cc,
@@ -323,7 +422,7 @@ pub async fn thread(
tags,
timestamp,
headers,
body,
body: final_body,
path,
attachments,
delivered_to,
@@ -397,14 +496,14 @@ fn email_addresses(
pub fn cid_attachment_bytes(nm: &Notmuch, id: &str, cid: &str) -> Result<Attachment, ServerError> {
let files = nm.files(id)?;
let Some(path) = files.first() else {
warn!("failed to find files for message {id}");
warn!("failed to find files for message {}", id);
return Err(ServerError::PartNotFound);
};
let file = File::open(&path)?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let m = parse_mail(&mmap)?;
if let Some(attachment) = walk_attachments(&m, |sp, _cur_idx| {
info!("{cid} {:?}", get_content_id(&sp.headers));
info!("{} {:?}", cid, get_content_id(&sp.headers));
if let Some(h_cid) = get_content_id(&sp.headers) {
let h_cid = &h_cid[1..h_cid.len() - 1];
if h_cid == cid {
@@ -425,7 +524,7 @@ pub fn cid_attachment_bytes(nm: &Notmuch, id: &str, cid: &str) -> Result<Attachm
pub fn attachment_bytes(nm: &Notmuch, id: &str, idx: &[usize]) -> Result<Attachment, ServerError> {
let files = nm.files(id)?;
let Some(path) = files.first() else {
warn!("failed to find files for message {id}");
warn!("failed to find files for message {}", id);
return Err(ServerError::PartNotFound);
};
let file = File::open(&path)?;
@@ -459,7 +558,7 @@ fn extract_body(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Ser
_ => extract_unhandled(m),
};
if let Err(err) = ret {
error!("Failed to extract body: {err:?}");
error!("Failed to extract body: {:?}", err);
return Ok(extract_unhandled(m)?);
}
ret
@@ -557,14 +656,20 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
},
contact_info: tlsrpt.contact_info.unwrap_or_else(|| "".to_string()),
report_id: tlsrpt.report_id,
policies: tlsrpt.policies.into_iter().map(|policy| {
FormattedTlsRptPolicy {
policies: tlsrpt
.policies
.into_iter()
.map(|policy| FormattedTlsRptPolicy {
policy: FormattedTlsRptPolicyDetails {
policy_type: policy.policy.policy_type,
policy_string: policy.policy.policy_string,
policy_domain: policy.policy.policy_domain,
mx_host: policy.policy.mx_host.unwrap_or_else(|| Vec::new()).into_iter().map(|mx| {
match mx {
mx_host: policy
.policy
.mx_host
.unwrap_or_else(|| Vec::new())
.into_iter()
.map(|mx| match mx {
MxHost::String(s) => FormattedTlsRptMxHost {
hostname: s,
failure_count: 0,
@@ -575,26 +680,42 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
failure_count: o.failure_count,
result_type: o.result_type,
},
}
}).collect(),
})
.collect(),
},
summary: policy.summary,
failure_details: policy.failure_details.unwrap_or_else(|| Vec::new()).into_iter().map(|detail| {
FormattedTlsRptFailureDetails {
failure_details: policy
.failure_details
.unwrap_or_else(|| Vec::new())
.into_iter()
.map(|detail| FormattedTlsRptFailureDetails {
result_type: detail.result_type,
sending_mta_ip: detail.sending_mta_ip.unwrap_or_else(|| "".to_string()),
receiving_ip: detail.receiving_ip.unwrap_or_else(|| "".to_string()),
receiving_mx_hostname: detail.receiving_mx_hostname.unwrap_or_else(|| "".to_string()),
sending_mta_ip: detail
.sending_mta_ip
.unwrap_or_else(|| "".to_string()),
receiving_ip: detail
.receiving_ip
.unwrap_or_else(|| "".to_string()),
receiving_mx_hostname: detail
.receiving_mx_hostname
.unwrap_or_else(|| "".to_string()),
failed_session_count: detail.failed_session_count,
additional_info: detail.additional_info.unwrap_or_else(|| "".to_string()),
failure_reason_code: detail.failure_reason_code.unwrap_or_else(|| "".to_string()),
}
}).collect(),
}
}).collect(),
additional_info: detail
.additional_info
.unwrap_or_else(|| "".to_string()),
failure_reason_code: detail
.failure_reason_code
.unwrap_or_else(|| "".to_string()),
})
.collect(),
})
.collect(),
};
let template = TlsReportTemplate { report: &formatted_tlsrpt };
template.render().unwrap_or_else(|e| format!("<div class=\"tlsrpt-error\">Failed to render TLS report template: {}</div>", e)) }
let template = TlsReportTemplate {
report: &formatted_tlsrpt,
};
template.render().unwrap_or_else(|e| format!("<div class=\"tlsrpt-error\">Failed to render TLS report template: {}</div>", e))
}
Err(e) => format!(
"<div class=\"tlsrpt-error\">Failed to parse TLS report JSON: {}</div>",
e
@@ -603,12 +724,10 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
} else {
format!("<div class=\"tlsrpt-error\">Failed to convert decompressed data to UTF-8.</div>")
}
}
else {
} else {
format!("<div class=\"tlsrpt-error\">Failed to decompressed data.</div>")
}
}
else {
} else {
"".to_string()
};
@@ -700,7 +819,10 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
.collect();
unhandled_types.sort();
if !unhandled_types.is_empty() {
warn!("{MULTIPART_MIXED} contains the following unhandled mimetypes {unhandled_types:?}");
warn!(
"{} contains the following unhandled mimetypes {:?}",
MULTIPART_MIXED, unhandled_types
);
}
let mut parts = Vec::new();
for (idx, sp) in m.subparts.iter().enumerate() {
@@ -751,8 +873,7 @@ fn unhandled_html(parent_type: &str, child_type: &str) -> Body {
Unhandled mimetype {} in a {} message
</div>
"#,
child_type,
parent_type
child_type, parent_type
),
content_tree: String::new(),
})
@@ -807,7 +928,10 @@ fn extract_related(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body,
.collect();
unhandled_types.sort();
if !unhandled_types.is_empty() {
warn!("{MULTIPART_RELATED} contains the following unhandled mimetypes {unhandled_types:?}");
warn!(
"{} contains the following unhandled mimetypes {:?}",
MULTIPART_RELATED, unhandled_types
);
}
for (i, sp) in m.subparts.iter().enumerate() {
@@ -820,7 +944,7 @@ fn extract_related(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body,
if let Some(cid) = sp.headers.get_first_value("Content-Id") {
let mut part_id = part_addr.clone();
part_id.push(i.to_string());
info!("cid: {cid} part_id {part_id:?}");
info!("cid: {} part_id {:?}", cid, part_id);
}
}
}
@@ -908,7 +1032,7 @@ fn extract_attachment(m: &ParsedMail, id: &str, idx: &[usize]) -> Option<Attachm
// No known filename, assume it's not an attachment
_ => return None,
};
info!("filename {filename}");
info!("filename {}", filename);
// TODO: grab this from somewhere
let content_id = None;
@@ -940,7 +1064,7 @@ fn email_address_strings(emails: &[Email]) -> Vec<String> {
emails
.iter()
.map(|e| e.to_string())
.inspect(|e| info!("e {e}"))
.inspect(|e| info!("e {}", e))
.collect()
}
@@ -964,11 +1088,7 @@ CC: {}
Date: {}
Subject: {}
"#,
from,
to,
cc,
date,
subject
from, to, cc, date, subject
);
Ok(Body::text(text))
}
@@ -981,7 +1101,7 @@ Subject: {}
}
pub fn get_attachment_filename(header_value: &str) -> &str {
info!("get_attachment_filename {header_value}");
info!("get_attachment_filename {}", header_value);
// Strip last "
let v = &header_value[..header_value.len() - 1];
if let Some(idx) = v.rfind('"') {
@@ -1071,7 +1191,7 @@ pub async fn set_read_status<'ctx>(
.iter()
.filter(|uid| is_notmuch_thread_or_id(uid))
.collect();
info!("set_read_status({unread} {uids:?})");
info!("set_read_status({} {:?})", unread, uids);
for uid in uids {
if unread {
nm.tag_add("unread", uid)?;
@@ -1086,10 +1206,11 @@ async fn photo_url_for_email_address(
pool: &PgPool,
addr: &str,
) -> Result<Option<String>, ServerError> {
let row = sqlx::query_as::<_, (String,)>(include_str!("../sql/photo_url_for_email_address.sql"))
.bind(addr)
.fetch_optional(pool)
.await?;
let row =
sqlx::query_as::<_, (String,)>(include_str!("../sql/photo_url_for_email_address.sql"))
.bind(addr)
.fetch_optional(pool)
.await?;
Ok(row.map(|r| r.0))
}
@@ -1114,14 +1235,17 @@ pub async fn label_unprocessed(
use futures::StreamExt;
let ids = nm.message_ids(query)?;
info!(
"Processing {limit:?} of {} messages with '{query}'",
ids.len()
"Processing {:?} of {} messages with '{}'",
limit,
ids.len(),
query
);
let rules: Vec<_> = sqlx::query_as::<_, (Json<Rule>,)>(include_str!("../sql/label_unprocessed.sql"))
.fetch(pool)
.map(|r| r.unwrap().0.0)
.collect()
.await;
let rules: Vec<_> =
sqlx::query_as::<_, (Json<Rule>,)>(include_str!("../sql/label_unprocessed.sql"))
.fetch(pool)
.map(|r| r.unwrap().0 .0)
.collect()
.await;
/*
use letterbox_shared::{Match, MatchType};
let rules = vec![Rule {
@@ -1146,7 +1270,7 @@ pub async fn label_unprocessed(
let files = nm.files(&id)?;
// Only process the first file path is multiple files have the same id
let Some(path) = files.iter().next() else {
error!("No files for message-ID {id}");
error!("No files for message-ID {}", id);
let t = "Letterbox/Bad";
nm.tag_add(t, &id)?;
let t = "unprocessed";
@@ -1154,12 +1278,12 @@ pub async fn label_unprocessed(
continue;
};
let file = File::open(&path)?;
info!("parsing {path}");
info!("parsing {}", path);
let mmap = unsafe { MmapOptions::new().map(&file)? };
let m = match info_span!("parse_mail", path = path).in_scope(|| parse_mail(&mmap)) {
Ok(m) => m,
Err(err) => {
error!("Failed to parse {path}: {err}");
error!("Failed to parse {}: {}", path, err);
let t = "Letterbox/Bad";
nm.tag_add(t, &id)?;
let t = "unprocessed";
@@ -1171,7 +1295,8 @@ pub async fn label_unprocessed(
if matched_rule {
if dryrun {
info!(
"\nAdd tags: {add_tags:?}\nTo: {} From: {} Subject: {}\n",
"\nAdd tags: {:?}\nTo: {} From: {} Subject: {}\n",
add_tags,
m.headers.get_first_value("to").expect("no from header"),
m.headers.get_first_value("from").expect("no from header"),
m.headers
@@ -1203,8 +1328,7 @@ pub async fn label_unprocessed(
.push(id.clone());
}
//nm.tag_remove("unprocessed", &id)?;
}
else {
} else {
if add_tags.is_empty() {
let t = "Grey".to_string();
add_mutations
@@ -1227,7 +1351,7 @@ pub async fn label_unprocessed(
}
info!("Adding {} distinct labels", add_mutations.len());
for (tag, ids) in add_mutations.iter() {
info!(" {tag}: {}", ids.len());
info!(" {}: {}", tag, ids.len());
if !dryrun {
let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect();
info_span!("tags_add", tag = tag, count = ids.len())
@@ -1236,7 +1360,7 @@ pub async fn label_unprocessed(
}
info!("Removing {} distinct labels", rm_mutations.len());
for (tag, ids) in rm_mutations.iter() {
info!(" {tag}: {}", ids.len());
info!(" {}: {}", tag, ids.len());
if !dryrun {
let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect();
info_span!("tags_remove", tag = tag, count = ids.len())
@@ -1252,7 +1376,7 @@ fn find_tags<'a, 'b>(rules: &'a [Rule], headers: &'b [MailHeader]) -> (bool, Has
for rule in rules {
for hdr in headers {
if rule.is_match(&hdr.get_key(), &hdr.get_value()) {
//info!("Matched {rule:?}");
//info!("Matched {:?}", rule);
matched_rule = true;
add_tags.insert(rule.tag.as_str());
if rule.stop_on_match {
@@ -1804,7 +1928,13 @@ mod tests {
let report: TlsRpt = serde_json::from_str(REPORT_V2).unwrap();
assert_eq!(report.organization_name, "Google Inc.");
assert_eq!(report.policies.len(), 1);
let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap();
let mx_host = report.policies[0]
.policy
.mx_host
.as_ref()
.unwrap()
.get(0)
.unwrap();
match mx_host {
MxHost::String(s) => assert_eq!(s, "mail.xinu.tv"),
MxHost::Object(_) => panic!("Expected a string"),
@@ -1816,13 +1946,19 @@ mod tests {
let report: TlsRpt = serde_json::from_str(REPORT_V3).unwrap();
assert_eq!(report.organization_name, "Google Inc.");
assert_eq!(report.policies.len(), 1);
let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap();
let mx_host = report.policies[0]
.policy
.mx_host
.as_ref()
.unwrap()
.get(0)
.unwrap();
match mx_host {
MxHost::Object(o) => {
assert_eq!(o.hostname, "mail.xinu.tv");
assert_eq!(o.failure_count, 0);
assert_eq!(o.result_type, "success");
},
}
MxHost::String(_) => panic!("Expected an object"),
}
}