server: more TLS report support and minor refactoring

server/sql/label_unprocessed.sql

@@ -0,0 +1 @@
+SELECT rule as "rule: Json<Rule>" FROM email_rule ORDER BY sort_order

server/sql/photo_url_for_email_address.sql

@@ -0,0 +1 @@
+SELECT url FROM email_photo ep JOIN email_address ea ON ep.id = ea.email_photo_id WHERE address = $1

server/src/nm.rs

@@ -563,7 +563,20 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
                                         policy_type: policy.policy.policy_type,
                                         policy_string: policy.policy.policy_string,
                                         policy_domain: policy.policy.policy_domain,
-                                        mx_host: policy.policy.mx_host.unwrap_or_else(|| Vec::new()),
+                                        mx_host: policy.policy.mx_host.unwrap_or_else(|| Vec::new()).into_iter().map(|mx| {
+                                            match mx {
+                                                MxHost::String(s) => FormattedTlsRptMxHost {
+                                                    hostname: s,
+                                                    failure_count: 0,
+                                                    result_type: "".to_string(),
+                                                },
+                                                MxHost::Object(o) => FormattedTlsRptMxHost {
+                                                    hostname: o.hostname,
+                                                    failure_count: o.failure_count,
+                                                    result_type: o.result_type,
+                                                },
+                                            }
+                                        }).collect(),
                                     },
                                     summary: policy.summary,
                                     failure_details: policy.failure_details.unwrap_or_else(|| Vec::new()).into_iter().map(|detail| {
@@ -581,8 +594,7 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
                             }).collect(),
                         };
                         let template = TlsReportTemplate { report: &formatted_tlsrpt };
-                        template.render().unwrap_or_else(|e| format!("<div class=\"tlsrpt-error\">Failed to render TLS report template: {}</div>", e))
+                        template.render().unwrap_or_else(|e| format!("<div class=\"tlsrpt-error\">Failed to render TLS report template: {}</div>", e))                    }
-                    }
                     Err(e) => format!(
                         "<div class=\"tlsrpt-error\">Failed to parse TLS report JSON: {}</div>",
                         e
@@ -591,10 +603,12 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec<String>) -> Result<Body,
             } else {
                 format!("<div class=\"tlsrpt-error\">Failed to convert decompressed data to UTF-8.</div>")
             }
-        } else {
+        }
+        else {
             format!("<div class=\"tlsrpt-error\">Failed to decompressed data.</div>")
         }
-    } else {
+    }
+    else {
         "".to_string()
     };
 
@@ -709,14 +723,15 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
                     // TODO: make URL generation more programatic based on what the frontend has
                     // mapped
                     parts.push(Body::html(format!(
-                        r#"<img src="/api/view/attachment/{}/{}/{filename}">"#,
+                        r#"<img src="/api/view/attachment/{}/{}/{}">"#,
                         part_addr[0],
                         part_addr
                             .iter()
                             .skip(1)
                             .map(|i| i.to_string())
                             .collect::<Vec<_>>()
-                            .join(".")
+                            .join("."),
+                        filename
                     )));
                 }
             }
@@ -733,9 +748,11 @@ fn unhandled_html(parent_type: &str, child_type: &str) -> Body {
         html: format!(
             r#"
 <div class="p-4 error">
-Unhandled mimetype {child_type} in a {parent_type} message
+Unhandled mimetype {} in a {} message
 </div>
-    "#
+    "#,
+            child_type,
+            parent_type
         ),
         content_tree: String::new(),
     })
@@ -898,7 +915,7 @@ fn extract_attachment(m: &ParsedMail, id: &str, idx: &[usize]) -> Option<Attachm
     let bytes = match m.get_body_raw() {
         Ok(bytes) => bytes,
         Err(err) => {
-            error!("failed to get body for attachment: {err}");
+            error!("failed to get body for attachment: {}", err);
             return None;
         }
     };
@@ -941,12 +958,17 @@ fn extract_rfc822(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, S
         let text = format!(
             r#"
 ---------- Forwarded message ----------
-From: {from}
+From: {}
-To: {to}
+To: {}
-CC: {cc}
+CC: {}
-Date: {date}
+Date: {}
-Subject: {subject}
+Subject: {}
-"#
+"#,
+            from,
+            to,
+            cc,
+            date,
+            subject
         );
         Ok(Body::text(text))
     }
@@ -1007,13 +1029,13 @@ fn render_content_type_tree(m: &ParsedMail) -> String {
         parts.push(msg);
         let indent = " ".repeat(depth * WIDTH);
         if !m.ctype.charset.is_empty() {
-            parts.push(format!("{indent} Character Set: {}", m.ctype.charset));
+            parts.push(format!("{} Character Set: {}", indent, m.ctype.charset));
         }
         for (k, v) in m.ctype.params.iter() {
-            parts.push(format!("{indent} {k}: {v}"));
+            parts.push(format!("{} {}: {}", indent, k, v));
         }
         if !m.headers.is_empty() {
-            parts.push(format!("{indent} == headers =="));
+            parts.push(format!("{} == headers ==", indent));
             for h in &m.headers {
                 if h.get_key().starts_with('X') {
                     continue;
@@ -1022,7 +1044,7 @@ fn render_content_type_tree(m: &ParsedMail) -> String {
                     continue;
                 }
 
-                parts.push(format!("{indent} {}: {}", h.get_key_ref(), h.get_value()));
+                parts.push(format!("{} {}: {}", indent, h.get_key_ref(), h.get_value()));
             }
         }
         for sp in &m.subparts {
@@ -1064,21 +1086,11 @@ async fn photo_url_for_email_address(
     pool: &PgPool,
     addr: &str,
 ) -> Result<Option<String>, ServerError> {
-    let row = sqlx::query!(
+    let row = sqlx::query_as::<_, (String,)>(include_str!("../sql/photo_url_for_email_address.sql"))
-        r#"
+    .bind(addr)
-SELECT
-  url
-FROM email_photo ep
-JOIN email_address ea
-ON ep.id = ea.email_photo_id
-WHERE
-  address = $1
-    "#,
-        addr
-    )
     .fetch_optional(pool)
     .await?;
-    Ok(row.map(|r| r.url))
+    Ok(row.map(|r| r.0))
 }
 
 /*
@@ -1105,15 +1117,9 @@ pub async fn label_unprocessed(
         "Processing {limit:?} of {} messages with '{query}'",
         ids.len()
     );
-    let rules: Vec<_> = sqlx::query!(
+    let rules: Vec<_> = sqlx::query_as::<_, (Json<Rule>,)>(include_str!("../sql/label_unprocessed.sql"))
-        r#"
-        SELECT rule as "rule: Json<Rule>"
-        FROM email_rule
-        ORDER BY sort_order
-        "#,
-    )
     .fetch(pool)
-    .map(|r| r.unwrap().rule.0)
+    .map(|r| r.unwrap().0.0)
     .collect()
     .await;
     /*
@@ -1136,7 +1142,7 @@ pub async fn label_unprocessed(
     let mut add_mutations = HashMap::new();
     let mut rm_mutations = HashMap::new();
     for id in ids {
-        let id = format!("id:{id}");
+        let id = format!("id:{}", id);
         let files = nm.files(&id)?;
         // Only process the first file path is multiple files have the same id
         let Some(path) = files.iter().next() else {
@@ -1197,7 +1203,8 @@ pub async fn label_unprocessed(
                     .push(id.clone());
             }
             //nm.tag_remove("unprocessed", &id)?;
-        } else {
+        }
+        else {
             if add_tags.is_empty() {
                 let t = "Grey".to_string();
                 add_mutations
@@ -1405,7 +1412,7 @@ pub struct TlsRptPolicyDetails {
     #[serde(rename = "policy-domain")]
     pub policy_domain: String,
     #[serde(rename = "mx-host")]
-    pub mx_host: Option<Vec<TlsRptMxHost>>,
+    pub mx_host: Option<Vec<MxHost>>,
 }
 
 #[derive(Debug, serde::Deserialize)]
@@ -1416,6 +1423,13 @@ pub struct TlsRptSummary {
     pub total_failure_session_count: u64,
 }
 
+#[derive(Debug, serde::Deserialize)]
+#[serde(untagged)]
+pub enum MxHost {
+    String(String),
+    Object(TlsRptMxHost),
+}
+
 #[derive(Debug, serde::Deserialize)]
 pub struct TlsRptMxHost {
     pub hostname: String,
@@ -1470,7 +1484,14 @@ pub struct FormattedTlsRptPolicyDetails {
     pub policy_type: String,
     pub policy_string: Vec<String>,
     pub policy_domain: String,
-    pub mx_host: Vec<TlsRptMxHost>,
+    pub mx_host: Vec<FormattedTlsRptMxHost>,
+}
+
+#[derive(Debug)]
+pub struct FormattedTlsRptMxHost {
+    pub hostname: String,
+    pub failure_count: u64,
+    pub result_type: String,
 }
 
 #[derive(Debug)]
@@ -1665,3 +1686,144 @@ pub fn parse_dmarc_report(xml: &str) -> Result<String, ServerError> {
     let html = template.render()?;
     Ok(html)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const REPORT_V1: &str = r#"
+    {
+        "organization-name": "Google Inc.",
+        "date-range": {
+            "start-datetime": "2025-08-09T00:00:00Z",
+            "end-datetime": "2025-08-09T23:59:59Z"
+        },
+        "contact-info": "smtp-tls-reporting@google.com",
+        "report-id": "2025-08-09T00:00:00Z_xinu.tv",
+        "policies": [
+            {
+                "policy": {
+                    "policy-type": "sts",
+                    "policy-string": [
+                        "version: STSv1",
+                        "mode: testing",
+                        "mx: mail.xinu.tv",
+                        "max_age: 86400"
+                    ],
+                    "policy-domain": "xinu.tv"
+                },
+                "summary": {
+                    "total-successful-session-count": 20,
+                    "total-failure-session-count": 0
+                }
+            }
+        ]
+    }
+    "#;
+
+    const REPORT_V2: &str = r#"
+    {
+        "organization-name": "Google Inc.",
+        "date-range": {
+            "start-datetime": "2025-08-09T00:00:00Z",
+            "end-datetime": "2025-08-09T23:59:59Z"
+        },
+        "contact-info": "smtp-tls-reporting@google.com",
+        "report-id": "2025-08-09T00:00:00Z_xinu.tv",
+        "policies": [
+            {
+                "policy": {
+                    "policy-type": "sts",
+                    "policy-string": [
+                        "version: STSv1",
+                        "mode: testing",
+                        "mx: mail.xinu.tv",
+                        "max_age: 86400"
+                    ],
+                    "policy-domain": "xinu.tv",
+                    "mx-host": [
+                        "mail.xinu.tv"
+                    ]
+                },
+                "summary": {
+                    "total-successful-session-count": 3,
+                    "total-failure-session-count": 0
+                }
+            }
+        ]
+    }
+    "#;
+
+    const REPORT_V3: &str = r#"
+    {
+        "organization-name": "Google Inc.",
+        "date-range": {
+            "start-datetime": "2025-08-09T00:00:00Z",
+            "end-datetime": "2025-08-09T23:59:59Z"
+        },
+        "contact-info": "smtp-tls-reporting@google.com",
+        "report-id": "2025-08-09T00:00:00Z_xinu.tv",
+        "policies": [
+            {
+                "policy": {
+                    "policy-type": "sts",
+                    "policy-string": [
+                        "version: STSv1",
+                        "mode: testing",
+                        "mx: mail.xinu.tv",
+                        "max_age: 86400"
+                    ],
+                    "policy-domain": "xinu.tv",
+                    "mx-host": [
+                        {
+                            "hostname": "mail.xinu.tv",
+                            "failure-count": 0,
+                            "result-type": "success"
+                        }
+                    ]
+                },
+                "summary": {
+                    "total-successful-session-count": 3,
+                    "total-failure-session-count": 0
+                }
+            }
+        ]
+    }
+    "#;
+
+    #[test]
+    fn test_parse_tls_report_v1() {
+        let report: TlsRpt = serde_json::from_str(REPORT_V1).unwrap();
+        assert_eq!(report.organization_name, "Google Inc.");
+        assert_eq!(report.policies.len(), 1);
+        assert_eq!(report.policies[0].policy.mx_host.is_none(), true);
+    }
+
+    #[test]
+    fn test_parse_tls_report_v2() {
+        let report: TlsRpt = serde_json::from_str(REPORT_V2).unwrap();
+        assert_eq!(report.organization_name, "Google Inc.");
+        assert_eq!(report.policies.len(), 1);
+        let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap();
+        match mx_host {
+            MxHost::String(s) => assert_eq!(s, "mail.xinu.tv"),
+            MxHost::Object(_) => panic!("Expected a string"),
+        }
+    }
+
+    #[test]
+    fn test_parse_tls_report_v3() {
+        let report: TlsRpt = serde_json::from_str(REPORT_V3).unwrap();
+        assert_eq!(report.organization_name, "Google Inc.");
+        assert_eq!(report.policies.len(), 1);
+        let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap();
+        match mx_host {
+            MxHost::Object(o) => {
+                assert_eq!(o.hostname, "mail.xinu.tv");
+                assert_eq!(o.failure_count, 0);
+                assert_eq!(o.result_type, "success");
+            },
+            MxHost::String(_) => panic!("Expected an object"),
+        }
+    }
+}