From 5c42d045983c21b9f28783b9fe63c5470ff3b9fc Mon Sep 17 00:00:00 2001 From: Bill Thiede Date: Tue, 12 Aug 2025 16:08:21 -0700 Subject: [PATCH] server: pretty print raw TLSRPT and DMARC data --- Cargo.lock | 476 +++++++++++++++++++++++++++++++++++++++--- server/Cargo.toml | 1 + server/src/error.rs | 2 + server/src/graphql.rs | 16 ++ server/src/nm.rs | 270 ++++++++++++++++++------ 5 files changed, 672 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 633cc4b..e9a3ab0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1131,6 +1131,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation" version = "0.9.4" @@ -1355,6 +1361,23 @@ dependencies = [ "url", ] +[[package]] +name = "cssparser" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db8599a9761b371751fbf13e076fa03c6e1a78f8c5288e6ab9467f10a2322c1" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 0.4.8", + "matches", + "phf 0.8.0", + "proc-macro2", + "quote", + "smallvec 1.15.1", + "syn 1.0.109", +] + [[package]] name = "cssparser" version = "0.34.0" @@ -1364,7 +1387,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa 1.0.15", - "phf", + "phf 0.11.3", "smallvec 1.15.1", ] @@ -1377,7 +1400,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa 1.0.15", - "phf", + "phf 0.11.3", "smallvec 1.15.1", ] @@ -1546,8 +1569,10 @@ version = "0.99.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ + "convert_case", "proc-macro2", "quote", + "rustc_version 0.4.1", "syn 2.0.104", ] @@ -1751,7 +1776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -3226,7 +3251,7 @@ dependencies = [ "mailparse", "maplit", "memmap", - "quick-xml", + "quick-xml 0.38.1", "regex", "reqwest", "scraper", @@ -3240,6 +3265,7 @@ dependencies = [ "tracing", "url", "urlencoding", + "xmlem", "xtracing", "zip", ] @@ -3528,8 +3554,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" dependencies = [ "log", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "string_cache", "string_cache_codegen", "tendril", @@ -3577,6 +3603,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "matchit" version = "0.7.3" @@ -3849,6 +3881,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "7.1.3" @@ -4302,14 +4340,35 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros 0.8.0", + "phf_shared 0.8.0", + "proc-macro-hack", +] + [[package]] name = "phf" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", ] [[package]] @@ -4318,8 +4377,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", ] [[package]] @@ -4328,30 +4397,53 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", "rand 0.8.5", ] +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "phf_macros" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.104", ] +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher 0.3.11", +] + [[package]] name = "phf_shared" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher", + "siphasher 1.0.1", ] [[package]] @@ -4524,6 +4616,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.96" @@ -4565,6 +4663,48 @@ dependencies = [ "prost", ] +[[package]] +name = "qname" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fac552bc5de28a2d8ebb5d93d157b1eb3d1ca58f88ba2d1e2b931951cb52b29" +dependencies = [ + "qname-impl", + "qname-macro", +] + +[[package]] +name = "qname-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28cd0334d1ed0cb075cde40d45b9ee648fe79e69a18c3e2891f79c3ec788304" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "qname-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e673fdaa2dcfd19fac443c4a361cc2635b2304592cdc5352ef76bb8dd38333c" +dependencies = [ + "proc-macro2", + "qname-impl", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quick-xml" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.38.1" @@ -4662,6 +4802,20 @@ dependencies = [ "scheduled-thread-pool", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", + "rand_pcg", +] + [[package]] name = "rand" version = "0.8.5" @@ -4683,6 +4837,16 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -4703,6 +4867,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -4731,6 +4904,24 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rayon" version = "1.10.0" @@ -5178,7 +5369,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5371,6 +5562,24 @@ dependencies = [ "web-sys", ] +[[package]] +name = "selectors" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdea87c686be721aab36607728047801ee21561bfdbd6bf0da7ace2536d5879f" +dependencies = [ + "bitflags 1.3.2", + "cssparser 0.28.1", + "derive_more 0.99.20", + "fxhash", + "log", + "phf 0.8.0", + "phf_codegen 0.8.0", + "precomputed-hash", + "servo_arc 0.1.1", + "smallvec 1.15.1", +] + [[package]] name = "selectors" version = "0.26.0" @@ -5383,10 +5592,10 @@ dependencies = [ "fxhash", "log", "new_debug_unreachable", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "precomputed-hash", - "servo_arc", + "servo_arc 0.4.1", "smallvec 1.15.1", ] @@ -5402,10 +5611,10 @@ dependencies = [ "fxhash", "log", "new_debug_unreachable", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "precomputed-hash", - "servo_arc", + "servo_arc 0.4.1", "smallvec 1.15.1", ] @@ -5528,6 +5737,16 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "servo_arc" version = "0.4.1" @@ -5636,6 +5855,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "siphasher" version = "1.0.1" @@ -5944,7 +6169,7 @@ checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", "parking_lot 0.12.4", - "phf_shared", + "phf_shared 0.11.3", "precomputed-hash", "serde", ] @@ -5955,8 +6180,8 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", "proc-macro2", "quote", ] @@ -6993,6 +7218,189 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-ucd" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625b18f7601e1127504a20ae731dc3c7826d0e86d5f7fe3434f8137669240efd" +dependencies = [ + "unic-ucd-age", + "unic-ucd-bidi", + "unic-ucd-block", + "unic-ucd-case", + "unic-ucd-category", + "unic-ucd-common", + "unic-ucd-hangul", + "unic-ucd-ident", + "unic-ucd-name", + "unic-ucd-name_aliases", + "unic-ucd-normal", + "unic-ucd-segment", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-age" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8cfdfe71af46b871dc6af2c24fcd360e2f3392ee4c5111877f2947f311671c" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-bidi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-block" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b2a16f2d7ecd25325a1053ca5a66e7fa1b68911a65c5e97f8d2e1b236b6f1d7" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-case" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d98d6246a79bac6cf66beee01422bda7c882e11d837fa4969bfaaba5fdea6d3" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-category" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" +dependencies = [ + "matches", + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9b78b910beafa1aae5c59bf00877c6cece1c5db28a1241ad801e86cecdff4ad" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-hangul" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054" +dependencies = [ + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-ident" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-name" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8fc55a45b2531089dc1773bf60c1f104b38e434b774ffc37b9c29a9b0f492e" +dependencies = [ + "unic-char-property", + "unic-ucd-hangul", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-name_aliases" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7674212643087699ba247a63dd05f1204c7e4880ec9342e545a7cffcc6a46f" +dependencies = [ + "unic-char-property", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-normal" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-category", + "unic-ucd-hangul", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + [[package]] name = "unicase" version = "2.8.1" @@ -7330,8 +7738,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" dependencies = [ - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "string_cache", "string_cache_codegen", ] @@ -7731,6 +8139,22 @@ version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7" +[[package]] +name = "xmlem" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d7eadd176b9e994da5c27acd46001a82256afa9e49b1b56b37e117e63a0aee" +dependencies = [ + "cssparser 0.28.1", + "indexmap 1.9.3", + "once_cell", + "qname", + "quick-xml 0.22.0", + "selectors 0.23.0", + "slotmap", + "unic-ucd", +] + [[package]] name = "xtracing" version = "0.3.2" diff --git a/server/Cargo.toml b/server/Cargo.toml index b0541bd..5e35c01 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -54,6 +54,7 @@ urlencoding = "2.1.3" #xtracing = { path = "../../xtracing" } xtracing = { version = "0.3.2", registry = "xinu" } zip = "4.3.0" +xmlem = "0.1.0" [build-dependencies] build-info-build = "0.0.41" diff --git a/server/src/error.rs b/server/src/error.rs index e39c8ae..d48c64d 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -41,4 +41,6 @@ pub enum ServerError { InfaillibleError(#[from] Infallible), #[error("askama error: {0}")] AskamaError(#[from] askama::Error), + #[error("xml error: {0}")] + XmlError(#[from] quick_xml::Error), } diff --git a/server/src/graphql.rs b/server/src/graphql.rs index 771ac99..39a4af2 100644 --- a/server/src/graphql.rs +++ b/server/src/graphql.rs @@ -237,6 +237,22 @@ impl Body { content_tree: "".to_string(), }) } + + pub fn to_html(&self) -> Option { + match self { + Body::Html(h) => Some(h.html.clone()), + Body::PlainText(p) => Some(format!("
{}
", html_escape::encode_text(&p.text))), + Body::UnhandledContentType(u) => Some(format!("
{}
", html_escape::encode_text(&u.text))), + } + } + + pub fn to_html_content_tree(&self) -> Option { + match self { + Body::Html(h) => Some(h.content_tree.clone()), + Body::PlainText(p) => Some(p.content_tree.clone()), + Body::UnhandledContentType(u) => Some(u.content_tree.clone()), + } + } } #[derive(Debug, SimpleObject)] diff --git a/server/src/nm.rs b/server/src/nm.rs index e9694ab..4b82f9b 100644 --- a/server/src/nm.rs +++ b/server/src/nm.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fs::File, io::{Cursor, Read}, + str::FromStr, }; use askama::Template; @@ -13,6 +14,7 @@ use memmap::MmapOptions; use quick_xml::de::from_str as xml_from_str; use sqlx::{types::Json, PgPool}; use tracing::{error, info, info_span, instrument, warn}; +use xmlem::{display, Document}; use zip::ZipArchive; use crate::{ @@ -176,7 +178,7 @@ pub async fn thread( // display names (that default to the most commonly seen name). let mut messages = Vec::new(); for (path, id) in std::iter::zip(nm.files(&thread_id)?, nm.message_ids(&thread_id)?) { - let tags = nm.tags_for_query(&format!("id:{id}"))?; + let tags = nm.tags_for_query(&format!("id:{}", id))?; let file = File::open(&path)?; let mmap = unsafe { MmapOptions::new().map(&file)? }; let m = parse_mail(&mmap)?; @@ -314,8 +316,105 @@ pub async fn thread( .collect(); // TODO(wathiede): parse message and fill out attachments let attachments = extract_attachments(&m, &id)?; + + let mut final_body = body; + let mut raw_report_content: Option = None; + + // Append TLS report if available + if m.ctype.mimetype.as_str() == MULTIPART_REPORT { + if let Ok(Body::Html(_html_body)) = extract_report(&m, &mut part_addr) { + // Extract raw JSON for pretty printing + if let Some(sp) = m + .subparts + .iter() + .find(|sp| sp.ctype.mimetype.as_str() == "application/tlsrpt+gzip") + { + if let Ok(gz_bytes) = sp.get_body_raw() { + let mut decoder = flate2::read::GzDecoder::new(&gz_bytes[..]); + let mut buffer = Vec::new(); + if decoder.read_to_end(&mut buffer).is_ok() { + if let Ok(json_str) = String::from_utf8(buffer) { + raw_report_content = Some(json_str); + } + } + } + } + } + } + + // Append DMARC report if available + if m.ctype.mimetype.as_str() == APPLICATION_ZIP { + if let Ok(Body::Html(_html_body)) = extract_zip(&m) { + // Extract raw XML for pretty printing + if let Ok(zip_bytes) = m.get_body_raw() { + if let Ok(mut archive) = ZipArchive::new(Cursor::new(&zip_bytes)) { + for i in 0..archive.len() { + if let Ok(mut file) = archive.by_index(i) { + let name = file.name().to_lowercase(); + if name.ends_with(".xml") + && (name.contains("dmarc") || name.starts_with("google.com!")) + { + let mut xml = String::new(); + use std::io::Read; + if file.read_to_string(&mut xml).is_ok() { + raw_report_content = Some(xml); + } + } + } + } + } + } + } + } + + if m.ctype.mimetype.as_str() == APPLICATION_GZIP { + if let Ok(Body::Html(_html_body)) = extract_gzip(&m) { + // Extract raw XML for pretty printing + if let Ok(gz_bytes) = m.get_body_raw() { + let mut decoder = flate2::read::GzDecoder::new(&gz_bytes[..]); + let mut xml = String::new(); + use std::io::Read; + if decoder.read_to_string(&mut xml).is_ok() { + raw_report_content = Some(xml); + } + } + } + } + + if let Some(raw_content) = raw_report_content { + let pretty_printed_content = if m.ctype.mimetype.as_str() == MULTIPART_REPORT { + // Pretty print JSON + if let Ok(parsed_json) = serde_json::from_str::(&raw_content) { + serde_json::to_string_pretty(&parsed_json).unwrap_or(raw_content) + } else { + raw_content + } + } else { + // DMARC reports are XML + // Pretty print XML + let doc_result = Document::from_str(&raw_content); + if let Ok(doc) = doc_result { + doc.to_string_pretty_with_config(&display::Config::default_pretty()) + } else { + error!( + "Failed to parse XML for pretty printing: {:?}", + doc_result.unwrap_err() + ); + raw_content + } + }; + final_body = Body::Html(Html { + html: format!( + "{}\n
{}
", + final_body.to_html().unwrap_or_default(), + html_escape::encode_text(&pretty_printed_content) + ), + content_tree: final_body.to_html_content_tree().unwrap_or_default(), + }); + } + messages.push(Message { - id: format!("id:{id}"), + id: format!("id:{}", id), from, to, cc, @@ -323,7 +422,7 @@ pub async fn thread( tags, timestamp, headers, - body, + body: final_body, path, attachments, delivered_to, @@ -397,14 +496,14 @@ fn email_addresses( pub fn cid_attachment_bytes(nm: &Notmuch, id: &str, cid: &str) -> Result { let files = nm.files(id)?; let Some(path) = files.first() else { - warn!("failed to find files for message {id}"); + warn!("failed to find files for message {}", id); return Err(ServerError::PartNotFound); }; let file = File::open(&path)?; let mmap = unsafe { MmapOptions::new().map(&file)? }; let m = parse_mail(&mmap)?; if let Some(attachment) = walk_attachments(&m, |sp, _cur_idx| { - info!("{cid} {:?}", get_content_id(&sp.headers)); + info!("{} {:?}", cid, get_content_id(&sp.headers)); if let Some(h_cid) = get_content_id(&sp.headers) { let h_cid = &h_cid[1..h_cid.len() - 1]; if h_cid == cid { @@ -425,7 +524,7 @@ pub fn cid_attachment_bytes(nm: &Notmuch, id: &str, cid: &str) -> Result Result { let files = nm.files(id)?; let Some(path) = files.first() else { - warn!("failed to find files for message {id}"); + warn!("failed to find files for message {}", id); return Err(ServerError::PartNotFound); }; let file = File::open(&path)?; @@ -459,7 +558,7 @@ fn extract_body(m: &ParsedMail, part_addr: &mut Vec) -> Result extract_unhandled(m), }; if let Err(err) = ret { - error!("Failed to extract body: {err:?}"); + error!("Failed to extract body: {:?}", err); return Ok(extract_unhandled(m)?); } ret @@ -557,14 +656,20 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec) -> Result FormattedTlsRptMxHost { hostname: s, failure_count: 0, @@ -575,26 +680,42 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec) -> ResultFailed to render TLS report template: {}", e)) } + let template = TlsReportTemplate { + report: &formatted_tlsrpt, + }; + template.render().unwrap_or_else(|e| format!("
Failed to render TLS report template: {}
", e)) + } Err(e) => format!( "
Failed to parse TLS report JSON: {}
", e @@ -603,12 +724,10 @@ fn extract_report(m: &ParsedMail, _part_addr: &mut Vec) -> ResultFailed to convert decompressed data to UTF-8.") } - } - else { + } else { format!("
Failed to decompressed data.
") } - } - else { + } else { "".to_string() }; @@ -700,7 +819,10 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec) -> Result Body { Unhandled mimetype {} in a {} message "#, - child_type, - parent_type + child_type, parent_type ), content_tree: String::new(), }) @@ -807,7 +928,10 @@ fn extract_related(m: &ParsedMail, part_addr: &mut Vec) -> Result) -> Result Option return None, }; - info!("filename {filename}"); + info!("filename {}", filename); // TODO: grab this from somewhere let content_id = None; @@ -940,7 +1064,7 @@ fn email_address_strings(emails: &[Email]) -> Vec { emails .iter() .map(|e| e.to_string()) - .inspect(|e| info!("e {e}")) + .inspect(|e| info!("e {}", e)) .collect() } @@ -964,11 +1088,7 @@ CC: {} Date: {} Subject: {} "#, - from, - to, - cc, - date, - subject + from, to, cc, date, subject ); Ok(Body::text(text)) } @@ -981,7 +1101,7 @@ Subject: {} } pub fn get_attachment_filename(header_value: &str) -> &str { - info!("get_attachment_filename {header_value}"); + info!("get_attachment_filename {}", header_value); // Strip last " let v = &header_value[..header_value.len() - 1]; if let Some(idx) = v.rfind('"') { @@ -1071,7 +1191,7 @@ pub async fn set_read_status<'ctx>( .iter() .filter(|uid| is_notmuch_thread_or_id(uid)) .collect(); - info!("set_read_status({unread} {uids:?})"); + info!("set_read_status({} {:?})", unread, uids); for uid in uids { if unread { nm.tag_add("unread", uid)?; @@ -1086,10 +1206,11 @@ async fn photo_url_for_email_address( pool: &PgPool, addr: &str, ) -> Result, ServerError> { - let row = sqlx::query_as::<_, (String,)>(include_str!("../sql/photo_url_for_email_address.sql")) - .bind(addr) - .fetch_optional(pool) - .await?; + let row = + sqlx::query_as::<_, (String,)>(include_str!("../sql/photo_url_for_email_address.sql")) + .bind(addr) + .fetch_optional(pool) + .await?; Ok(row.map(|r| r.0)) } @@ -1114,14 +1235,17 @@ pub async fn label_unprocessed( use futures::StreamExt; let ids = nm.message_ids(query)?; info!( - "Processing {limit:?} of {} messages with '{query}'", - ids.len() + "Processing {:?} of {} messages with '{}'", + limit, + ids.len(), + query ); - let rules: Vec<_> = sqlx::query_as::<_, (Json,)>(include_str!("../sql/label_unprocessed.sql")) - .fetch(pool) - .map(|r| r.unwrap().0.0) - .collect() - .await; + let rules: Vec<_> = + sqlx::query_as::<_, (Json,)>(include_str!("../sql/label_unprocessed.sql")) + .fetch(pool) + .map(|r| r.unwrap().0 .0) + .collect() + .await; /* use letterbox_shared::{Match, MatchType}; let rules = vec![Rule { @@ -1146,7 +1270,7 @@ pub async fn label_unprocessed( let files = nm.files(&id)?; // Only process the first file path is multiple files have the same id let Some(path) = files.iter().next() else { - error!("No files for message-ID {id}"); + error!("No files for message-ID {}", id); let t = "Letterbox/Bad"; nm.tag_add(t, &id)?; let t = "unprocessed"; @@ -1154,12 +1278,12 @@ pub async fn label_unprocessed( continue; }; let file = File::open(&path)?; - info!("parsing {path}"); + info!("parsing {}", path); let mmap = unsafe { MmapOptions::new().map(&file)? }; let m = match info_span!("parse_mail", path = path).in_scope(|| parse_mail(&mmap)) { Ok(m) => m, Err(err) => { - error!("Failed to parse {path}: {err}"); + error!("Failed to parse {}: {}", path, err); let t = "Letterbox/Bad"; nm.tag_add(t, &id)?; let t = "unprocessed"; @@ -1171,7 +1295,8 @@ pub async fn label_unprocessed( if matched_rule { if dryrun { info!( - "\nAdd tags: {add_tags:?}\nTo: {} From: {} Subject: {}\n", + "\nAdd tags: {:?}\nTo: {} From: {} Subject: {}\n", + add_tags, m.headers.get_first_value("to").expect("no from header"), m.headers.get_first_value("from").expect("no from header"), m.headers @@ -1203,8 +1328,7 @@ pub async fn label_unprocessed( .push(id.clone()); } //nm.tag_remove("unprocessed", &id)?; - } - else { + } else { if add_tags.is_empty() { let t = "Grey".to_string(); add_mutations @@ -1227,7 +1351,7 @@ pub async fn label_unprocessed( } info!("Adding {} distinct labels", add_mutations.len()); for (tag, ids) in add_mutations.iter() { - info!(" {tag}: {}", ids.len()); + info!(" {}: {}", tag, ids.len()); if !dryrun { let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect(); info_span!("tags_add", tag = tag, count = ids.len()) @@ -1236,7 +1360,7 @@ pub async fn label_unprocessed( } info!("Removing {} distinct labels", rm_mutations.len()); for (tag, ids) in rm_mutations.iter() { - info!(" {tag}: {}", ids.len()); + info!(" {}: {}", tag, ids.len()); if !dryrun { let ids: Vec<_> = ids.iter().map(|s| s.as_str()).collect(); info_span!("tags_remove", tag = tag, count = ids.len()) @@ -1252,7 +1376,7 @@ fn find_tags<'a, 'b>(rules: &'a [Rule], headers: &'b [MailHeader]) -> (bool, Has for rule in rules { for hdr in headers { if rule.is_match(&hdr.get_key(), &hdr.get_value()) { - //info!("Matched {rule:?}"); + //info!("Matched {:?}", rule); matched_rule = true; add_tags.insert(rule.tag.as_str()); if rule.stop_on_match { @@ -1804,7 +1928,13 @@ mod tests { let report: TlsRpt = serde_json::from_str(REPORT_V2).unwrap(); assert_eq!(report.organization_name, "Google Inc."); assert_eq!(report.policies.len(), 1); - let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap(); + let mx_host = report.policies[0] + .policy + .mx_host + .as_ref() + .unwrap() + .get(0) + .unwrap(); match mx_host { MxHost::String(s) => assert_eq!(s, "mail.xinu.tv"), MxHost::Object(_) => panic!("Expected a string"), @@ -1816,13 +1946,19 @@ mod tests { let report: TlsRpt = serde_json::from_str(REPORT_V3).unwrap(); assert_eq!(report.organization_name, "Google Inc."); assert_eq!(report.policies.len(), 1); - let mx_host = report.policies[0].policy.mx_host.as_ref().unwrap().get(0).unwrap(); + let mx_host = report.policies[0] + .policy + .mx_host + .as_ref() + .unwrap() + .get(0) + .unwrap(); match mx_host { MxHost::Object(o) => { assert_eq!(o.hostname, "mail.xinu.tv"); assert_eq!(o.failure_count, 0); assert_eq!(o.result_type, "success"); - }, + } MxHost::String(_) => panic!("Expected an object"), } }