server: add ability to slurp contents from site

From:
https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/html/resources/html.css

.cargo/config.toml

@@ -0,0 +1,4 @@
+
+[build]
+rustflags = [ "--cfg=web_sys_unstable_apis" ]
+

Cargo.lock

@@ -72,7 +72,7 @@ version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170"
 dependencies = [
- "html5ever",
+ "html5ever 0.26.0",
  "maplit",
  "once_cell",
  "tendril",
@@ -272,6 +272,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
 [[package]]
 name = "atomic_hooks"
 version = "0.1.6"
@@ -328,6 +334,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "base64ct"
 version = "1.6.0"
@@ -586,11 +598,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0250ac93bbccb4f0a892507a4580178edddef5e8267650e294b4fe00597b0da8"
 dependencies = [
  "cssparser 0.31.2",
- "html5ever",
+ "html5ever 0.26.0",
  "indexmap 2.2.6",
  "pico-args",
  "rayon",
- "reqwest",
+ "reqwest 0.11.27",
  "rustc-hash",
  "selectors 0.25.0",
  "smallvec",
@@ -838,6 +850,12 @@ dependencies = [
  "paste",
 ]
 
+[[package]]
+name = "ego-tree"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642"
+
 [[package]]
 name = "either"
 version = "1.13.0"
@@ -941,6 +959,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -1102,6 +1135,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.1.16"
@@ -1316,6 +1358,25 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "h2"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http 1.1.0",
+ "indexmap 2.2.6",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
 [[package]]
 name = "handlebars"
 version = "4.5.0"
@@ -1427,6 +1488,15 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "html-escape"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
+dependencies = [
+ "utf8-width",
+]
+
 [[package]]
 name = "html5ever"
 version = "0.26.0"
@@ -1435,12 +1505,26 @@ checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"
 dependencies = [
  "log",
  "mac",
- "markup5ever",
+ "markup5ever 0.11.0",
  "proc-macro2",
  "quote",
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "html5ever"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever 0.12.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.69",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -1474,6 +1558,29 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http 1.1.0",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "httparse"
 version = "1.9.4"
@@ -1502,9 +1609,9 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
+ "h2 0.3.26",
  "http 0.2.12",
- "http-body",
+ "http-body 0.4.6",
  "httparse",
  "httpdate",
  "itoa 1.0.11",
@@ -1516,6 +1623,26 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2 0.4.6",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "httparse",
+ "itoa 1.0.11",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
 [[package]]
 name = "hyper-rustls"
 version = "0.24.2"
@@ -1524,10 +1651,63 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
  "futures-util",
  "http 0.2.12",
- "hyper",
+ "hyper 0.14.29",
- "rustls",
+ "rustls 0.21.12",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.24.1",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
+dependencies = [
+ "futures-util",
+ "http 1.1.0",
+ "hyper 1.4.1",
+ "hyper-util",
+ "rustls 0.23.12",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls 0.26.0",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
+dependencies = [
+ "bytes",
+ "http-body-util",
+ "hyper 1.4.1",
+ "hyper-util",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "hyper 1.4.1",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower",
+ "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -1846,6 +2026,20 @@ dependencies = [
  "tendril",
 ]
 
+[[package]]
+name = "markup5ever"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45"
+dependencies = [
+ "log",
+ "phf 0.11.2",
+ "phf_codegen 0.11.2",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
@@ -1965,6 +2159,23 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "new_debug_unreachable"
 version = "1.0.6"
@@ -2091,6 +2302,50 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "openssl"
+version = "0.10.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.69",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "owning_ref"
 version = "0.4.1"
@@ -2287,6 +2542,16 @@ dependencies = [
  "phf_shared 0.10.0",
 ]
 
+[[package]]
+name = "phf_codegen"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a"
+dependencies = [
+ "phf_generator 0.11.2",
+ "phf_shared 0.11.2",
+]
+
 [[package]]
 name = "phf_generator"
 version = "0.8.0"
@@ -2751,11 +3016,11 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2",
+ "h2 0.3.26",
  "http 0.2.12",
- "http-body",
+ "http-body 0.4.6",
- "hyper",
+ "hyper 0.14.29",
- "hyper-rustls",
+ "hyper-rustls 0.24.2",
  "ipnet",
  "js-sys",
  "log",
@@ -2763,15 +3028,15 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls",
+ "rustls 0.21.12",
- "rustls-pemfile",
+ "rustls-pemfile 1.0.4",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 0.1.2",
- "system-configuration",
+ "system-configuration 0.5.1",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.24.1",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -2781,6 +3046,50 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "reqwest"
+version = "0.12.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "encoding_rs",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2 0.4.6",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.4.1",
+ "hyper-rustls 0.27.2",
+ "hyper-tls",
+ "hyper-util",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls-pemfile 2.1.3",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper 1.0.1",
+ "system-configuration 0.6.1",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "windows-registry",
+]
+
 [[package]]
 name = "ring"
 version = "0.17.8"
@@ -2878,7 +3187,7 @@ dependencies = [
  "either",
  "futures",
  "http 0.2.12",
- "hyper",
+ "hyper 0.14.29",
  "indexmap 2.2.6",
  "log",
  "memchr",
@@ -2957,10 +3266,23 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
 dependencies = [
  "log",
  "ring",
- "rustls-webpki",
+ "rustls-webpki 0.101.7",
  "sct",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.23.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044"
+dependencies = [
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki 0.102.6",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "rustls-pemfile"
 version = "1.0.4"
@@ -2970,6 +3292,22 @@ dependencies = [
  "base64 0.21.7",
 ]
 
+[[package]]
+name = "rustls-pemfile"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
+dependencies = [
+ "base64 0.22.1",
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
+
 [[package]]
 name = "rustls-webpki"
 version = "0.101.7"
@@ -2980,6 +3318,17 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "rustls-webpki"
+version = "0.102.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.17"
@@ -2992,6 +3341,15 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
+[[package]]
+name = "schannel"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "scoped-tls"
 version = "1.0.1"
@@ -3004,6 +3362,22 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "scraper"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b90460b31bfe1fc07be8262e42c665ad97118d4585869de9345a84d501a9eaf0"
+dependencies = [
+ "ahash 0.8.11",
+ "cssparser 0.31.2",
+ "ego-tree",
+ "getopts",
+ "html5ever 0.27.0",
+ "once_cell",
+ "selectors 0.25.0",
+ "tendril",
+]
+
 [[package]]
 name = "sct"
 version = "0.7.1"
@@ -3014,6 +3388,29 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "security-framework"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
+dependencies = [
+ "bitflags 2.6.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "seed"
 version = "0.10.0"
@@ -3166,8 +3563,10 @@ dependencies = [
  "anyhow",
  "async-graphql",
  "async-graphql-rocket",
+ "async-trait",
  "css-inline",
  "glog",
+ "html-escape",
  "linkify",
  "log",
  "lol_html",
@@ -3175,8 +3574,10 @@ dependencies = [
  "maplit",
  "memmap",
  "notmuch",
+ "reqwest 0.12.7",
  "rocket",
  "rocket_cors",
+ "scraper",
  "serde",
  "serde_json",
  "shared",
@@ -3666,6 +4067,15 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
+[[package]]
+name = "sync_wrapper"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+dependencies = [
+ "futures-core",
+]
+
 [[package]]
 name = "system-configuration"
 version = "0.5.1"
@@ -3674,7 +4084,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
 dependencies = [
  "bitflags 1.2.1",
  "core-foundation",
- "system-configuration-sys",
+ "system-configuration-sys 0.5.0",
+]
+
+[[package]]
+name = "system-configuration"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
+dependencies = [
+ "bitflags 2.6.0",
+ "core-foundation",
+ "system-configuration-sys 0.6.0",
 ]
 
 [[package]]
@@ -3687,6 +4108,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "system-configuration-sys"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.10.1"
@@ -3829,13 +4260,34 @@ dependencies = [
  "syn 2.0.69",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "rustls",
+ "rustls 0.21.12",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+dependencies = [
+ "rustls 0.23.12",
+ "rustls-pki-types",
  "tokio",
 ]
 
@@ -3932,6 +4384,27 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project",
+ "pin-project-lite",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
 [[package]]
 name = "tower-service"
 version = "0.3.2"
@@ -4089,6 +4562,12 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.4"
@@ -4140,10 +4619,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
 
 [[package]]
-name = "uuid"
+name = "utf8-width"
-version = "1.9.1"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439"
+checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
+
+[[package]]
+name = "uuid"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
 dependencies = [
  "getrandom 0.2.15",
 ]
@@ -4377,6 +4862,36 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-registry"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
+dependencies = [
+ "windows-result",
+ "windows-strings",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"

Cargo.toml

@@ -1,5 +1,8 @@
 [workspace]
 resolver = "2"
+default-members = [
+  "server"
+]
 members = [
   "web",
   "server",

notmuch/src/lib.rs

@@ -518,7 +518,8 @@ impl Notmuch {
     }
 
     pub fn count(&self, query: &str) -> Result<usize, NotmuchError> {
-        let res = self.run_notmuch(["count", query])?;
+        // TODO: compare speed of notmuch count for * w/ and w/o --output=threads
+        let res = self.run_notmuch(["count", "--output=threads", query])?;
         // Strip '\n' from res.
         let s = std::str::from_utf8(&res[..res.len() - 1])?;
         Ok(s.parse()?)

server/.sqlx/query-113694cd5bf0d2582ff3a635776daa608fe88abe1185958c4215646c92335afb.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    date,\n    is_read,\n    link,\n    site,\n    summary,\n    title,\n    name,\n    homepage\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    uid = $1\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "date",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_read",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "link",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "summary",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "homepage",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "113694cd5bf0d2582ff3a635776daa608fe88abe1185958c4215646c92335afb"
+}

server/.sqlx/query-2c1954b6db3cbcabf9b878cd1c8ea01c607f46dc43a85b58e19217e7633cf337.json

@@ -0,0 +1,55 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    site,\n    date,\n    is_read,\n    title,\n    uid,\n    name\nFROM\n    post p\n    JOIN feed f ON p.site = f.slug\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\nORDER BY\n    date DESC,\n    title OFFSET $3\nLIMIT\n    $4\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "date",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_read",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "uid",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "2c1954b6db3cbcabf9b878cd1c8ea01c607f46dc43a85b58e19217e7633cf337"
+}

server/.sqlx/query-2dcbedef656e1b725c5ba4fb67d31ce7962d8714449b2fb630f49a7ed1acc270.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    site,\n    name,\n    count (\n        NOT is_read\n        OR NULL\n    ) unread\nFROM\n    post AS p\n    JOIN feed AS f ON p.site = f.slug --\n    -- TODO: figure this out to make the query faster when only looking for unread\n    --WHERE\n    --    (\n    --        NOT $1\n    --        OR NOT is_read\n    --    )\nGROUP BY\n    1,\n    2\nORDER BY\n    site\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "site",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "unread",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true,
+      true,
+      null
+    ]
+  },
+  "hash": "2dcbedef656e1b725c5ba4fb67d31ce7962d8714449b2fb630f49a7ed1acc270"
+}

server/.sqlx/query-b39147b9d06171cb742141eda4675688cb702fb284758b1224ed3aa2d7f3b3d9.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE\n    post\nSET\n    is_read = $1\nWHERE\n    uid = $2\n",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b39147b9d06171cb742141eda4675688cb702fb284758b1224ed3aa2d7f3b3d9"
+}

server/.sqlx/query-e28b890e308f483aa6bd08617548ae66294ae1e99b1cab49f5f4211e0fd7d419.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n    COUNT(*) count\nFROM\n    post\nWHERE\n    ($1::text IS NULL OR site = $1)\n    AND (\n        NOT $2\n        OR NOT is_read\n    )\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e28b890e308f483aa6bd08617548ae66294ae1e99b1cab49f5f4211e0fd7d419"
+}

server/Cargo.toml

@@ -7,27 +7,30 @@ default-run = "server"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rocket = { version = "0.5.0-rc.2", features = [ "json" ] }
+ammonia = "3.3.0"
-notmuch = { path = "../notmuch" }
+anyhow = "1.0.79"
-shared = { path = "../shared" }
-serde_json = "1.0.87"
-thiserror = "1.0.37"
-serde = { version = "1.0.147", features = ["derive"] }
-log = "0.4.17"
-tokio = "1.26.0"
-glog = "0.1.0"
-urlencoding = "2.1.3"
 async-graphql = { version = "6.0.11", features = ["log"] }
 async-graphql-rocket = "6.0.11"
-rocket_cors = "0.6.0"
+async-trait = "0.1.81"
-memmap = "0.7.0"
-mailparse = "0.15.0"
-ammonia = "3.3.0"
-lol_html = "1.2.0"
 css-inline = "0.13.0"
-anyhow = "1.0.79"
+glog = "0.1.0"
-maplit = "1.0.2"
+html-escape = "0.2.13"
 linkify = "0.10.0"
+log = "0.4.17"
+lol_html = "1.2.0"
+mailparse = "0.15.0"
+maplit = "1.0.2"
+memmap = "0.7.0"
+notmuch = { path = "../notmuch" }
+reqwest = { version = "0.12.7", features = ["blocking"] }
+rocket = { version = "0.5.0-rc.2", features = [ "json" ] }
+rocket_cors = "0.6.0"
+scraper = "0.20.0"
+serde = { version = "1.0.147", features = ["derive"] }
+serde_json = "1.0.87"
+shared = { path = "../shared" }
 sqlx = { version = "0.7.4", features = ["postgres", "runtime-tokio", "time"] }
+thiserror = "1.0.37"
+tokio = "1.26.0"
 url = "2.5.2"
-
+urlencoding = "2.1.3"

server/sql/count.sql

@@ -3,7 +3,7 @@ SELECT
 FROM
     post
 WHERE
-    site = $1
+    ($1::text IS NULL OR site = $1)
     AND (
         NOT $2
         OR NOT is_read

server/sql/set_unread.sql

@@ -0,0 +1,6 @@
+UPDATE
+    post
+SET
+    is_read = $1
+WHERE
+    uid = $2

server/sql/threads.sql

@@ -1,4 +1,5 @@
 SELECT
+    site,
     date,
     is_read,
     title,
@@ -8,7 +9,7 @@ FROM
     post p
     JOIN feed f ON p.site = f.slug
 WHERE
-    site = $1
+    ($1::text IS NULL OR site = $1)
     AND (
         NOT $2
         OR NOT is_read

server/src/bin/cleanhtml.rs

@@ -9,7 +9,7 @@ fn main() -> anyhow::Result<()> {
     println!("Sanitizing {src} into {dst}");
     let bytes = fs::read(src)?;
     let html = String::from_utf8_lossy(&bytes);
-    let html = sanitize_html(&html, "")?;
+    let html = sanitize_html(&html, "", &None)?;
     fs::write(dst, html)?;
 
     Ok(())

server/src/custom.css

@@ -0,0 +1,8 @@
+pre {
+  background-color: var(--color-bg);
+  color: var(--color-text);
+}
+
+code {
+  background-color: var(--color-bg-secondary);
+}

server/src/error.rs

@@ -3,7 +3,7 @@ use std::{convert::Infallible, str::Utf8Error, string::FromUtf8Error};
 use mailparse::MailParseError;
 use thiserror::Error;
 
-use crate::SanitizeError;
+use crate::TransformError;
 
 #[derive(Error, Debug)]
 pub enum ServerError {
@@ -19,8 +19,8 @@ pub enum ServerError {
     PartNotFound,
     #[error("sqlx error: {0}")]
     SQLXError(#[from] sqlx::Error),
-    #[error("html sanitize error: {0}")]
+    #[error("html transform error: {0}")]
-    SanitizeError(#[from] SanitizeError),
+    TransformError(#[from] TransformError),
     #[error("UTF8 error: {0}")]
     Utf8Error(#[from] Utf8Error),
     #[error("FromUTF8 error: {0}")]

server/src/graphql.rs

@@ -1,13 +1,14 @@
-
 use async_graphql::{
-    connection::{Connection},
+    connection::{self, Connection, Edge, OpaqueCursor},
-    Context, EmptySubscription, Enum, Error, FieldResult, Object, Schema, SimpleObject, Union,
+    Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
+    SimpleObject, Union,
 };
 use log::info;
 use notmuch::Notmuch;
+use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPool;
 
-use crate::{newsreader, nm};
+use crate::{newsreader, nm, Query};
 
 /// # Number of seconds since the Epoch
 pub type UnixTime = isize;
@@ -194,13 +195,19 @@ pub struct Email {
 }
 
 #[derive(SimpleObject)]
-pub(crate) struct Tag {
+pub struct Tag {
     pub name: String,
     pub fg_color: String,
     pub bg_color: String,
     pub unread: usize,
 }
 
+#[derive(Serialize, Deserialize, Debug, InputObject)]
+struct SearchCursor {
+    newsreader_offset: i32,
+    notmuch_offset: i32,
+}
+
 pub struct QueryRoot;
 #[Object]
 impl QueryRoot {
@@ -208,12 +215,12 @@ impl QueryRoot {
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
 
-        // TODO: make this search both copra and merge results
+        let newsreader_query: Query = query.parse()?;
-        if newsreader::is_newsreader_search(&query) {
+
-            Ok(newsreader::count(pool, &query).await?)
+        let newsreader_count = newsreader::count(pool, &newsreader_query).await?;
-        } else {
+        let notmuch_count = nm::count(nm, &newsreader_query.to_notmuch()).await?;
-            Ok(nm::count(nm, &query).await?)
+        info!("count {newsreader_query:?} newsreader count {newsreader_count} notmuch count {notmuch_count}");
-        }
+        Ok(newsreader_count + notmuch_count)
     }
 
     async fn search<'ctx>(
@@ -224,17 +231,124 @@ impl QueryRoot {
         first: Option<i32>,
         last: Option<i32>,
         query: String,
-    ) -> Result<Connection<usize, ThreadSummary>, Error> {
+    ) -> Result<Connection<OpaqueCursor<SearchCursor>, ThreadSummary>, Error> {
-        info!("search({after:?} {before:?} {first:?} {last:?} {query:?})");
+        // TODO: add keywords to limit search to one corpus, i.e. is:news or is:mail
+        info!("search({after:?} {before:?} {first:?} {last:?} {query:?})",);
         let nm = ctx.data_unchecked::<Notmuch>();
         let pool = ctx.data_unchecked::<PgPool>();
 
-        // TODO: make this search both copra and merge results
+        enum ThreadSummaryCursor {
-        if newsreader::is_newsreader_search(&query) {
+            Newsreader(i32, ThreadSummary),
-            Ok(newsreader::search(pool, after, before, first, last, query).await?)
+            Notmuch(i32, ThreadSummary),
-        } else {
-            Ok(nm::search(nm, after, before, first, last, query).await?)
         }
+        Ok(connection::query(
+            after,
+            before,
+            first,
+            last,
+            |after: Option<OpaqueCursor<SearchCursor>>,
+             before: Option<OpaqueCursor<SearchCursor>>,
+             first: Option<usize>,
+             last: Option<usize>| async move {
+                info!(
+                    "search({:?} {:?} {first:?} {last:?} {query:?})",
+                    after.as_ref().map(|v| &v.0),
+                    before.as_ref().map(|v| &v.0)
+                );
+                let newsreader_after = after.as_ref().map(|sc| sc.newsreader_offset);
+                let notmuch_after = after.as_ref().map(|sc| sc.notmuch_offset);
+                let newsreader_before = before.as_ref().map(|sc| sc.newsreader_offset);
+                let notmuch_before = before.as_ref().map(|sc| sc.notmuch_offset);
+
+                let newsreader_query: Query = query.parse()?;
+                info!("newsreader_query {newsreader_query:?}");
+                let newsreader_results = if newsreader_query.is_newsreader {
+                    newsreader::search(
+                        pool,
+                        newsreader_after,
+                        newsreader_before,
+                        first.map(|v| v as i32),
+                        last.map(|v| v as i32),
+                        &newsreader_query,
+                    )
+                    .await?
+                    .into_iter()
+                    .map(|(cur, ts)| ThreadSummaryCursor::Newsreader(cur, ts))
+                    .collect()
+                } else {
+                    Vec::new()
+                };
+
+                let notmuch_results = if newsreader_query.is_notmuch {
+                    nm::search(
+                        nm,
+                        notmuch_after,
+                        notmuch_before,
+                        first.map(|v| v as i32),
+                        last.map(|v| v as i32),
+                        newsreader_query.to_notmuch(),
+                    )
+                    .await?
+                    .into_iter()
+                    .map(|(cur, ts)| ThreadSummaryCursor::Notmuch(cur, ts))
+                    .collect()
+                } else {
+                    Vec::new()
+                };
+
+                let mut results: Vec<_> = newsreader_results
+                    .into_iter()
+                    .chain(notmuch_results)
+                    .collect();
+
+                // The leading '-' is to reverse sort
+                results.sort_by_key(|item| match item {
+                    ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
+                    ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
+                });
+
+                let mut has_next_page = before.is_some();
+                if let Some(first) = first {
+                    if results.len() > first {
+                        has_next_page = true;
+                        results.truncate(first);
+                    }
+                }
+
+                let mut has_previous_page = after.is_some();
+                if let Some(last) = last {
+                    if results.len() > last {
+                        has_previous_page = true;
+                        results.truncate(last);
+                    }
+                }
+
+                let mut connection = Connection::new(has_previous_page, has_next_page);
+                let mut newsreader_offset = 0;
+                let mut notmuch_offset = 0;
+
+                connection.edges.extend(results.into_iter().map(|item| {
+                    let thread_summary;
+                    match item {
+                        ThreadSummaryCursor::Newsreader(offset, ts) => {
+                            thread_summary = ts;
+                            newsreader_offset = offset;
+                        }
+                        ThreadSummaryCursor::Notmuch(offset, ts) => {
+                            thread_summary = ts;
+                            notmuch_offset = offset;
+                        }
+                    }
+                    let cur = OpaqueCursor(SearchCursor {
+                        newsreader_offset,
+                        notmuch_offset,
+                    });
+                    Edge::new(cur, thread_summary)
+                }));
+                Ok::<_, async_graphql::Error>(connection)
+            },
+        )
+        .await?)
     }
 
     async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {
@@ -273,11 +387,14 @@ impl Mutation {
         unread: bool,
     ) -> Result<bool, Error> {
         let nm = ctx.data_unchecked::<Notmuch>();
-        info!("set_read_status({unread})");
+        let pool = ctx.data_unchecked::<PgPool>();
-        if unread {
+
-            nm.tag_add("unread", &format!("{query}"))?;
+        for q in query.split_whitespace() {
-        } else {
+            if newsreader::is_newsreader_thread(&q) {
-            nm.tag_remove("unread", &format!("{query}"))?;
+                newsreader::set_read_status(pool, &q, unread).await?;
+            } else {
+                nm::set_read_status(nm, q, unread).await?;
+            }
         }
         Ok(true)
     }

server/src/lib.rs

@@ -3,20 +3,202 @@ pub mod graphql;
 pub mod newsreader;
 pub mod nm;
 
+use std::{collections::HashMap, convert::Infallible, str::FromStr};
+
+use async_trait::async_trait;
 use css_inline::{CSSInliner, InlineError, InlineOptions};
 use linkify::{LinkFinder, LinkKind};
-use log::error;
+use log::{error, info, warn};
-use lol_html::{element, errors::RewritingError, rewrite_str, RewriteStrSettings};
+use lol_html::{element, errors::RewritingError, rewrite_str, text, RewriteStrSettings};
 use maplit::{hashmap, hashset};
+use scraper::{error::SelectorErrorKind, Html, Selector};
 use thiserror::Error;
 use url::Url;
 
+use crate::newsreader::{
+    extract_thread_id, is_newsreader_search, is_newsreader_thread, make_news_tag,
+};
+const NON_EXISTENT_SITE_NAME: &'static str = "NO-SUCH-SITE";
+
+// TODO: figure out how to use Cow
+#[async_trait]
+trait Transformer: Send + Sync {
+    fn should_run(&self, addr: &Option<Url>, _html: &str) -> bool {
+        true
+    }
+    // TODO: should html be something like `html_escape` uses:
+    // <S: ?Sized + AsRef<str>>(text: &S) -> Cow<str>
+    async fn transform(&self, addr: &Option<Url>, html: &str) -> Result<String, TransformError>;
+}
+
+// TODO: how would we make this more generic to allow good implementations of Transformer outside
+// of this module?
 #[derive(Error, Debug)]
-pub enum SanitizeError {
+pub enum TransformError {
-    #[error("lol-html rewrite error")]
+    #[error("lol-html rewrite error: {0}")]
     RewritingError(#[from] RewritingError),
-    #[error("css inline error")]
+    #[error("css inline error: {0}")]
     InlineError(#[from] InlineError),
+    #[error("failed to fetch url error: {0}")]
+    ReqwestError(#[from] reqwest::Error),
+    #[error("failed to parse HTML: {0}")]
+    HtmlParsingError(String),
+}
+
+struct SanitizeHtml<'a> {
+    cid_prefix: &'a str,
+    base_url: &'a Option<Url>,
+}
+
+#[async_trait]
+impl<'a> Transformer for SanitizeHtml<'a> {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        Ok(sanitize_html(html, self.cid_prefix, self.base_url)?)
+    }
+}
+
+struct EscapeHtml;
+
+#[async_trait]
+impl Transformer for EscapeHtml {
+    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+        html.contains("&")
+    }
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        Ok(html_escape::decode_html_entities(html).to_string())
+    }
+}
+
+struct StripHtml;
+
+#[async_trait]
+impl Transformer for StripHtml {
+    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+        // Lame test
+        html.contains("<")
+    }
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        let mut text = String::new();
+        let element_content_handlers = vec![text!("*", |t| {
+            text += t.as_str();
+            Ok(())
+        })];
+        let _ = rewrite_str(
+            html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
+
+        Ok(text)
+    }
+}
+
+struct InlineStyle;
+
+#[async_trait]
+impl Transformer for InlineStyle {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        let css = concat!(
+            "/* chrome-default.css */\n",
+            include_str!("chrome-default.css"),
+            "\n/* mvp.css */\n",
+            include_str!("mvp.css"),
+            "\n/* Xinu Specific overrides */\n",
+            include_str!("custom.css"),
+        );
+        let inline_opts = InlineOptions {
+            inline_style_tags: false,
+            keep_style_tags: false,
+            keep_link_tags: false,
+            base_url: None,
+            load_remote_stylesheets: false,
+            extra_css: Some(css.into()),
+            preallocate_node_capacity: 32,
+            ..InlineOptions::default()
+        };
+
+        Ok(match CSSInliner::new(inline_opts).inline(&html) {
+            Ok(inlined_html) => inlined_html,
+            Err(err) => {
+                error!("failed to inline CSS: {err}");
+                html.to_string()
+            }
+        })
+    }
+}
+
+struct AddOutlink;
+
+#[async_trait]
+impl Transformer for AddOutlink {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        if let Some(link) = link {
+            link.scheme().starts_with("http") && !html.contains(link.as_str())
+        } else {
+            false
+        }
+    }
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        if let Some(link) = link {
+            Ok(format!(
+                r#"
+                {html}
+                <div><a href="{}">View on site</a></div>
+                "#,
+                link
+            ))
+        } else {
+            Ok(html.to_string())
+        }
+    }
+}
+
+struct SlurpContents {
+    site_selectors: HashMap<String, Vec<Selector>>,
+}
+
+impl SlurpContents {
+    fn get_selectors(&self, link: &Url) -> Option<&[Selector]> {
+        for (host, selector) in self.site_selectors.iter() {
+            if link.host_str().map(|h| h.contains(host)).unwrap_or(false) {
+                return Some(&selector);
+            }
+        }
+        None
+    }
+}
+
+#[async_trait]
+impl Transformer for SlurpContents {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        if let Some(link) = link {
+            return self.get_selectors(link).is_some();
+        }
+        false
+    }
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        let Some(link) = link else {
+            return Ok(html.to_string());
+        };
+        let Some(selectors) = self.get_selectors(&link) else {
+            return Ok(html.to_string());
+        };
+        let body = reqwest::get(link.as_str()).await?.text().await?;
+        let doc = Html::parse_document(&body);
+
+        let mut results = Vec::new();
+        for selector in selectors {
+            if let Some(frag) = doc.select(&selector).next() {
+                results.push(frag.html())
+            } else {
+                warn!("couldn't find '{:?}' in {}", selector, link);
+                return Ok(html.to_string());
+            }
+        }
+        Ok(results.join("<br><br>"))
+    }
 }
 
 pub fn linkify_html(text: &str) -> String {
@@ -50,31 +232,15 @@ pub fn linkify_html(text: &str) -> String {
 pub fn sanitize_html(
     html: &str,
     cid_prefix: &str,
-    base_url: &Url,
+    base_url: &Option<Url>,
-) -> Result<String, SanitizeError> {
+) -> Result<String, TransformError> {
-    let element_content_handlers = vec![
+    let mut element_content_handlers = vec![
         // Open links in new tab
         element!("a[href]", |el| {
             el.set_attribute("target", "_blank").unwrap();
 
             Ok(())
         }),
-        // Make links with relative URLs absolute
-        element!("a[href]", |el| {
-            if let Some(Ok(href)) = el.get_attribute("href").map(|href| base_url.join(&href)) {
-                el.set_attribute("href", &href.as_str()).unwrap();
-            }
-
-            Ok(())
-        }),
-        // Make images with relative srcs absolute
-        element!("img[src]", |el| {
-            if let Some(Ok(src)) = el.get_attribute("src").map(|src| base_url.join(&src)) {
-                el.set_attribute("src", &src.as_str()).unwrap();
-            }
-
-            Ok(())
-        }),
         // Replace mixed part CID images with URL
         element!("img[src]", |el| {
             let src = el
@@ -98,6 +264,26 @@ pub fn sanitize_html(
             Ok(())
         }),
     ];
+    if let Some(base_url) = base_url {
+        element_content_handlers.extend(vec![
+            // Make links with relative URLs absolute
+            element!("a[href]", |el| {
+                if let Some(Ok(href)) = el.get_attribute("href").map(|href| base_url.join(&href)) {
+                    el.set_attribute("href", &href.as_str()).unwrap();
+                }
+
+                Ok(())
+            }),
+            // Make images with relative srcs absolute
+            element!("img[src]", |el| {
+                if let Some(Ok(src)) = el.get_attribute("src").map(|src| base_url.join(&src)) {
+                    el.set_attribute("src", &src.as_str()).unwrap();
+                }
+
+                Ok(())
+            }),
+        ]);
+    }
 
     let inline_opts = InlineOptions {
         inline_style_tags: true,
@@ -276,3 +462,119 @@ pub fn sanitize_html(
 
     Ok(clean_html)
 }
+
+fn compute_offset_limit(
+    after: Option<i32>,
+    before: Option<i32>,
+    first: Option<i32>,
+    last: Option<i32>,
+) -> (i32, i32) {
+    let default_page_size = 100;
+    match (after, before, first, last) {
+        // Reasonable defaults
+        (None, None, None, None) => (0, default_page_size),
+        (None, None, Some(first), None) => (0, first),
+        (Some(after), None, None, None) => (after + 1, default_page_size),
+        (Some(after), None, Some(first), None) => (after + 1, first),
+        (None, Some(before), None, None) => (0.max(before - default_page_size), default_page_size),
+        (None, Some(before), None, Some(last)) => (0.max(before - last), last),
+        (None, None, None, Some(_)) => {
+            panic!("specifying last and no before doesn't make sense")
+        }
+        (None, None, Some(_), Some(_)) => {
+            panic!("specifying first and last doesn't make sense")
+        }
+        (None, Some(_), Some(_), _) => {
+            panic!("specifying before and first doesn't make sense")
+        }
+        (Some(_), Some(_), _, _) => {
+            panic!("specifying after and before doesn't make sense")
+        }
+        (Some(_), None, None, Some(_)) => {
+            panic!("specifying after and last doesn't make sense")
+        }
+        (Some(_), None, Some(_), Some(_)) => {
+            panic!("specifying after, first and last doesn't make sense")
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Query {
+    pub unread_only: bool,
+    pub tag: Option<String>,
+    pub uid: Option<String>,
+    pub remainder: Vec<String>,
+    pub is_notmuch: bool,
+    pub is_newsreader: bool,
+}
+
+impl Query {
+    // Converts the internal state of Query to something suitable for notmuch queries. Removes and
+    // letterbox specific '<key>:<value' tags
+    fn to_notmuch(&self) -> String {
+        let mut parts = Vec::new();
+        if !self.is_notmuch {
+            return String::new();
+        }
+
+        if self.unread_only {
+            parts.push("is:unread".to_string());
+        }
+        if let Some(site) = &self.tag {
+            parts.push(format!("tag:{site}"));
+        }
+        if let Some(uid) = &self.uid {
+            parts.push(uid.clone());
+        }
+        parts.extend(self.remainder.clone());
+        parts.join(" ")
+    }
+}
+
+impl FromStr for Query {
+    type Err = Infallible;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut unread_only = false;
+        let mut tag = None;
+        let mut uid = None;
+        let mut remainder = Vec::new();
+        let site_prefix = make_news_tag("");
+        let mut is_notmuch = false;
+        let mut is_newsreader = false;
+        for word in s.split_whitespace() {
+            if word == "is:unread" {
+                unread_only = true
+            } else if word.starts_with("tag:") {
+                tag = Some(word["tag:".len()..].to_string())
+                /*
+                            } else if word.starts_with("tag:") {
+                                // Any tag that doesn't match site_prefix should explicitly set the site to something not in the
+                                // database
+                                site = Some(NON_EXISTENT_SITE_NAME.to_string());
+                */
+            } else if is_newsreader_thread(word) {
+                uid = Some(extract_thread_id(word).to_string())
+            } else if word == "is:mail" || word == "is:email" || word == "is:notmuch" {
+                is_notmuch = true;
+            } else if word == "is:news" || word == "is:newsreader" {
+                is_newsreader = true;
+            } else {
+                remainder.push(word.to_string());
+            }
+        }
+        // If we don't see any explicit filters for a corpus, flip them all on
+        if !(is_notmuch || is_newsreader) {
+            is_newsreader = true;
+            is_notmuch = true;
+        }
+        Ok(Query {
+            unread_only,
+            tag,
+            uid,
+            remainder,
+            is_notmuch,
+            is_newsreader,
+        })
+    }
+}

server/src/mvp.css

@@ -0,0 +1,498 @@
+/* MVP.css v1.15 - https://github.com/andybrewer/mvp */
+
+/* :root content stored in client side index.html */
+
+html {
+  scroll-behavior: smooth;
+}
+
+@media (prefers-reduced-motion: reduce) {
+  html {
+    scroll-behavior: auto;
+  }
+}
+
+/* Layout */
+article aside {
+  background: var(--color-secondary-accent);
+  border-left: 4px solid var(--color-secondary);
+  padding: 0.01rem 0.8rem;
+}
+
+body {
+  background: var(--color-bg);
+  color: var(--color-text);
+  font-family: var(--font-family);
+  line-height: var(--line-height);
+  margin: 0;
+  overflow-x: hidden;
+  padding: 0;
+}
+
+footer,
+header,
+main {
+  margin: 0 auto;
+  max-width: var(--width-content);
+  padding: 3rem 1rem;
+}
+
+hr {
+  background-color: var(--color-bg-secondary);
+  border: none;
+  height: 1px;
+  margin: 4rem 0;
+  width: 100%;
+}
+
+section {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: var(--justify-important);
+}
+
+section img,
+article img {
+  max-width: 100%;
+}
+
+section pre {
+  overflow: auto;
+}
+
+section aside {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  margin: 1rem;
+  padding: 1.25rem;
+  width: var(--width-card);
+}
+
+section aside:hover {
+  box-shadow: var(--box-shadow) var(--color-bg-secondary);
+}
+
+[hidden] {
+  display: none;
+}
+
+/* Headers */
+article header,
+div header,
+main header {
+  padding-top: 0;
+}
+
+header {
+  text-align: var(--justify-important);
+}
+
+header a b,
+header a em,
+header a i,
+header a strong {
+  margin-left: 0.5rem;
+  margin-right: 0.5rem;
+}
+
+header nav img {
+  margin: 1rem 0;
+}
+
+section header {
+  padding-top: 0;
+  width: 100%;
+}
+
+/* Nav */
+nav {
+  align-items: center;
+  display: flex;
+  font-weight: bold;
+  justify-content: space-between;
+  margin-bottom: 7rem;
+}
+
+nav ul {
+  list-style: none;
+  padding: 0;
+}
+
+nav ul li {
+  display: inline-block;
+  margin: 0 0.5rem;
+  position: relative;
+  text-align: left;
+}
+
+/* Nav Dropdown */
+nav ul li:hover ul {
+  display: block;
+}
+
+nav ul li ul {
+  background: var(--color-bg);
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  display: none;
+  height: auto;
+  left: -2px;
+  padding: .5rem 1rem;
+  position: absolute;
+  top: 1.7rem;
+  white-space: nowrap;
+  width: auto;
+  z-index: 1;
+}
+
+nav ul li ul::before {
+  /* fill gap above to make mousing over them easier */
+  content: "";
+  position: absolute;
+  left: 0;
+  right: 0;
+  top: -0.5rem;
+  height: 0.5rem;
+}
+
+nav ul li ul li,
+nav ul li ul li a {
+  display: block;
+}
+
+/* Typography */
+code,
+samp {
+  background-color: var(--color-accent);
+  border-radius: var(--border-radius);
+  color: var(--color-text);
+  display: inline-block;
+  margin: 0 0.1rem;
+  padding: 0 0.5rem;
+}
+
+details {
+  margin: 1.3rem 0;
+}
+
+details summary {
+  font-weight: bold;
+  cursor: pointer;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  line-height: var(--line-height);
+  text-wrap: balance;
+}
+
+mark {
+  padding: 0.1rem;
+}
+
+ol li,
+ul li {
+  padding: 0.2rem 0;
+}
+
+p {
+  margin: 0.75rem 0;
+  padding: 0;
+  width: 100%;
+}
+
+pre {
+  margin: 1rem 0;
+  max-width: var(--width-card-wide);
+  padding: 1rem 0;
+}
+
+pre code,
+pre samp {
+  display: block;
+  max-width: var(--width-card-wide);
+  padding: 0.5rem 2rem;
+  white-space: pre-wrap;
+}
+
+small {
+  color: var(--color-text-secondary);
+}
+
+sup {
+  background-color: var(--color-secondary);
+  border-radius: var(--border-radius);
+  color: var(--color-bg);
+  font-size: xx-small;
+  font-weight: bold;
+  margin: 0.2rem;
+  padding: 0.2rem 0.3rem;
+  position: relative;
+  top: -2px;
+}
+
+/* Links */
+a {
+  color: var(--color-link);
+  display: inline-block;
+  font-weight: bold;
+  text-decoration: underline;
+}
+
+a:hover {
+  filter: brightness(var(--hover-brightness));
+}
+
+a:active {
+  filter: brightness(var(--active-brightness));
+}
+
+a b,
+a em,
+a i,
+a strong,
+button,
+input[type="submit"] {
+  border-radius: var(--border-radius);
+  display: inline-block;
+  font-size: medium;
+  font-weight: bold;
+  line-height: var(--line-height);
+  margin: 0.5rem 0;
+  padding: 1rem 2rem;
+}
+
+button,
+input[type="submit"] {
+  font-family: var(--font-family);
+}
+
+button:hover,
+input[type="submit"]:hover {
+  cursor: pointer;
+  filter: brightness(var(--hover-brightness));
+}
+
+button:active,
+input[type="submit"]:active {
+  filter: brightness(var(--active-brightness));
+}
+
+a b,
+a strong,
+button,
+input[type="submit"] {
+  background-color: var(--color-link);
+  border: 2px solid var(--color-link);
+  color: var(--color-bg);
+}
+
+a em,
+a i {
+  border: 2px solid var(--color-link);
+  border-radius: var(--border-radius);
+  color: var(--color-link);
+  display: inline-block;
+  padding: 1rem 2rem;
+}
+
+article aside a {
+  color: var(--color-secondary);
+}
+
+/* Images */
+figure {
+  margin: 0;
+  padding: 0;
+}
+
+figure img {
+  max-width: 100%;
+}
+
+figure figcaption {
+  color: var(--color-text-secondary);
+}
+
+/* Forms */
+button:disabled,
+input:disabled {
+  background: var(--color-bg-secondary);
+  border-color: var(--color-bg-secondary);
+  color: var(--color-text-secondary);
+  cursor: not-allowed;
+}
+
+button[disabled]:hover,
+input[type="submit"][disabled]:hover {
+  filter: none;
+}
+
+form {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  display: block;
+  max-width: var(--width-card-wide);
+  min-width: var(--width-card);
+  padding: 1.5rem;
+  text-align: var(--justify-normal);
+}
+
+form header {
+  margin: 1.5rem 0;
+  padding: 1.5rem 0;
+}
+
+input,
+label,
+select,
+textarea {
+  display: block;
+  font-size: inherit;
+  max-width: var(--width-card-wide);
+}
+
+input[type="checkbox"],
+input[type="radio"] {
+  display: inline-block;
+}
+
+input[type="checkbox"]+label,
+input[type="radio"]+label {
+  display: inline-block;
+  font-weight: normal;
+  position: relative;
+  top: 1px;
+}
+
+input[type="range"] {
+  padding: 0.4rem 0;
+}
+
+input,
+select,
+textarea {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  margin-bottom: 1rem;
+  padding: 0.4rem 0.8rem;
+}
+
+input[type="text"],
+input[type="password"] textarea {
+  width: calc(100% - 1.6rem);
+}
+
+input[readonly],
+textarea[readonly] {
+  background-color: var(--color-bg-secondary);
+}
+
+label {
+  font-weight: bold;
+  margin-bottom: 0.2rem;
+}
+
+/* Popups */
+dialog {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  box-shadow: var(--box-shadow) var(--color-shadow);
+  position: fixed;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  width: 50%;
+  z-index: 999;
+}
+
+/* Tables */
+table {
+  border: 1px solid var(--color-bg-secondary);
+  border-radius: var(--border-radius);
+  border-spacing: 0;
+  display: inline-block;
+  max-width: 100%;
+  overflow-x: auto;
+  padding: 0;
+  white-space: nowrap;
+}
+
+table td,
+table th,
+table tr {
+  padding: 0.4rem 0.8rem;
+  text-align: var(--justify-important);
+}
+
+table thead {
+  background-color: var(--color-table);
+  border-collapse: collapse;
+  border-radius: var(--border-radius);
+  color: var(--color-bg);
+  margin: 0;
+  padding: 0;
+}
+
+table thead tr:first-child th:first-child {
+  border-top-left-radius: var(--border-radius);
+}
+
+table thead tr:first-child th:last-child {
+  border-top-right-radius: var(--border-radius);
+}
+
+table thead th:first-child,
+table tr td:first-child {
+  text-align: var(--justify-normal);
+}
+
+table tr:nth-child(even) {
+  background-color: var(--color-accent);
+}
+
+/* Quotes */
+blockquote {
+  display: block;
+  font-size: x-large;
+  line-height: var(--line-height);
+  margin: 1rem auto;
+  max-width: var(--width-card-medium);
+  padding: 1.5rem 1rem;
+  text-align: var(--justify-important);
+}
+
+blockquote footer {
+  color: var(--color-text-secondary);
+  display: block;
+  font-size: small;
+  line-height: var(--line-height);
+  padding: 1.5rem 0;
+}
+
+/* Scrollbars */
+* {
+  scrollbar-width: thin;
+  scrollbar-color: var(--color-scrollbar) transparent;
+}
+
+*::-webkit-scrollbar {
+  width: 5px;
+  height: 5px;
+}
+
+*::-webkit-scrollbar-track {
+  background: transparent;
+}
+
+*::-webkit-scrollbar-thumb {
+  background-color: var(--color-scrollbar);
+  border-radius: 10px;
+}

server/src/newsreader.rs

@@ -1,21 +1,21 @@
-use std::{
+use std::hash::{DefaultHasher, Hash, Hasher};
-    convert::Infallible,
-    hash::{DefaultHasher, Hash, Hasher},
-    str::FromStr,
-};
 
-use async_graphql::connection::{self, Connection, Edge};
 use log::info;
+use maplit::hashmap;
+use scraper::Selector;
 use sqlx::postgres::PgPool;
 use url::Url;
 
+use crate::Query;
+
 const TAG_PREFIX: &'static str = "News/";
 const THREAD_PREFIX: &'static str = "news:";
 
 use crate::{
+    compute_offset_limit,
     error::ServerError,
     graphql::{Body, Email, Html, Message, Tag, Thread, ThreadSummary},
-    sanitize_html,
+    AddOutlink, EscapeHtml, InlineStyle, SanitizeHtml, SlurpContents, StripHtml, Transformer,
 };
 
 pub fn is_newsreader_search(query: &str) -> bool {
@@ -26,10 +26,26 @@ pub fn is_newsreader_thread(query: &str) -> bool {
     query.starts_with(THREAD_PREFIX)
 }
 
-pub async fn count(pool: &PgPool, query: &str) -> Result<usize, ServerError> {
+pub fn extract_thread_id(query: &str) -> &str {
-    let query: Query = query.parse()?;
+    &query[THREAD_PREFIX.len()..]
-    let site = query.site.expect("search has no site");
+}
-    let row = sqlx::query_file!("sql/count.sql", site, query.unread_only)
+
+pub fn extract_site(tag: &str) -> &str {
+    &tag[TAG_PREFIX.len()..]
+}
+
+pub fn make_news_tag(tag: &str) -> String {
+    format!("tag:{TAG_PREFIX}{tag}")
+}
+
+pub async fn count(pool: &PgPool, query: &Query) -> Result<usize, ServerError> {
+    if !query.remainder.is_empty() {
+        // TODO: handle full text search against all sites, for now, early return if search words
+        // are specified.
+        return Ok(0);
+    }
+
+    let row = sqlx::query_file!("sql/count.sql", query.tag, query.unread_only)
         .fetch_one(pool)
         .await?;
     Ok(row.count.unwrap_or(0).try_into().unwrap_or(0))
@@ -37,105 +53,72 @@ pub async fn count(pool: &PgPool, query: &str) -> Result<usize, ServerError> {
 
 pub async fn search(
     pool: &PgPool,
-    after: Option<String>,
+    after: Option<i32>,
-    before: Option<String>,
+    before: Option<i32>,
     first: Option<i32>,
     last: Option<i32>,
-    query: String,
+    query: &Query,
-) -> Result<Connection<usize, ThreadSummary>, async_graphql::Error> {
+) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
-    let query: Query = query.parse()?;
+    info!("search({after:?} {before:?} {first:?} {last:?} {query:?}");
-    info!("news search query {query:?}");
+    if !query.remainder.is_empty() {
-    let site = query.site.expect("search has no site");
+        // TODO: handle full text search against all sites, for now, early return if search words
-    connection::query(
+        // are specified.
-        after,
+        return Ok(Vec::new());
-        before,
+    }
-        first,
-        last,
-        |after: Option<usize>, before: Option<usize>, first, last| async move {
-            info!("search page info {after:#?}, {before:#?}, {first:#?}, {last:#?}");
-            let default_page_size = 100;
-            let (offset, limit) = match (after, before, first, last) {
-                // Reasonable defaults
-                (None, None, None, None) => (0, default_page_size),
-                (None, None, Some(first), None) => (0, first),
-                (Some(after), None, None, None) => (after, default_page_size),
-                (Some(after), None, Some(first), None) => (after, first),
-                (None, Some(before), None, None) => {
-                    (before.saturating_sub(default_page_size), default_page_size)
-                }
-                (None, Some(before), None, Some(last)) => (before.saturating_sub(last), last),
-                (None, None, None, Some(_)) => {
-                    panic!("specifying last and no before doesn't make sense")
-                }
-                (None, None, Some(_), Some(_)) => {
-                    panic!("specifying first and last doesn't make sense")
-                }
-                (None, Some(_), Some(_), _) => {
-                    panic!("specifying before and first doesn't make sense")
-                }
-                (Some(_), Some(_), _, _) => {
-                    panic!("specifying after and before doesn't make sense")
-                }
-                (Some(_), None, None, Some(_)) => {
-                    panic!("specifying after and last doesn't make sense")
-                }
-                (Some(_), None, Some(_), Some(_)) => {
-                    panic!("specifying after, first and last doesn't make sense")
-                }
-            };
-            // The +1 is to see if there are more pages of data available.
-            let limit = limit + 1;
 
-            info!("search page offset {offset} limit {limit}");
+    let (offset, mut limit) = compute_offset_limit(after, before, first, last);
-            let rows = sqlx::query_file!(
+    if before.is_none() {
-                "sql/threads.sql",
+        // When searching forward, the +1 is to see if there are more pages of data available.
-                site,
+        // Searching backwards implies there's more pages forward, because the value represented by
-                query.unread_only,
+        // `before` is on the next page.
-                offset as i64,
+        limit = limit + 1;
-                limit as i64
+    }
-            )
-            .fetch_all(pool)
-            .await?;
 
-            let mut slice = rows
+    let site = query.tag.as_ref().map(|t| extract_site(&t).to_string());
-                .into_iter()
+    info!(
-                .map(|r| {
+        "search offset {offset} limit {limit} site {site:?} unread_only {}",
-                    let tags = if r.is_read.unwrap_or(false) {
+        query.unread_only
-                        vec![site.clone()]
+    );
-                    } else {
+
-                        vec!["unread".to_string(), site.clone()]
+    // TODO: further limit results to include query.remainder if set
-                    };
+    let rows = sqlx::query_file!(
-                    ThreadSummary {
+        "sql/threads.sql",
-                        thread: format!("{THREAD_PREFIX}{}", r.uid),
+        site,
-                        timestamp: r
+        query.unread_only,
-                            .date
+        offset as i64,
-                            .expect("post missing date")
+        limit as i64
-                            .assume_utc()
-                            .unix_timestamp() as isize,
-                        date_relative: "TODO date_relative".to_string(),
-                        matched: 0,
-                        total: 1,
-                        authors: r.name.unwrap_or_else(|| site.clone()),
-                        subject: r.title.unwrap_or("NO TITLE".to_string()),
-                        tags,
-                    }
-                })
-                .collect::<Vec<_>>();
-            let has_more = slice.len() == limit;
-            let mut connection = Connection::new(offset > 0, has_more);
-            if has_more {
-                slice.pop();
-            };
-            connection.edges.extend(
-                slice
-                    .into_iter()
-                    .enumerate()
-                    .map(|(idx, item)| Edge::new(offset + idx, item)),
-            );
-            Ok::<_, async_graphql::Error>(connection)
-        },
     )
-    .await
+    .fetch_all(pool)
+    .await?;
+
+    let mut res = Vec::new();
+    for (i, r) in rows.into_iter().enumerate() {
+        let site = r.site.unwrap_or("UNKOWN TAG".to_string());
+        let mut tags = vec![format!("{TAG_PREFIX}{site}")];
+        if !r.is_read.unwrap_or(true) {
+            tags.push("unread".to_string());
+        };
+        let mut title = r.title.unwrap_or("NO TITLE".to_string());
+        title = clean_title(&title).await.expect("failed to clean title");
+        res.push((
+            i as i32 + offset,
+            ThreadSummary {
+                thread: format!("{THREAD_PREFIX}{}", r.uid),
+                timestamp: r
+                    .date
+                    .expect("post missing date")
+                    .assume_utc()
+                    .unix_timestamp() as isize,
+                date_relative: "TODO date_relative".to_string(),
+                matched: 0,
+                total: 1,
+                authors: r.name.unwrap_or_else(|| site.clone()),
+                subject: title,
+                tags,
+            },
+        ));
+    }
+    Ok(res)
 }
 
 pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
@@ -170,11 +153,10 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
         .fetch_one(pool)
         .await?;
 
-    let site = r.site.unwrap_or("NO SITE".to_string());
+    let site = r.site.unwrap_or("NO TAG".to_string());
-    let tags = if r.is_read.unwrap_or(false) {
+    let mut tags = vec![format!("{TAG_PREFIX}{site}")];
-        vec![site.clone()]
+    if r.is_read.unwrap_or(true) {
-    } else {
+        tags.push("unread".to_string());
-        vec!["unread".to_string(), site.clone()]
     };
     let default_homepage = "http://no-homepage";
     let homepage = Url::parse(
@@ -188,18 +170,18 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
             })
             .unwrap_or(default_homepage.to_string()),
     )?;
-    let link = Url::parse(
+    let link = &r
-        &r.link
+        .link
-            .as_ref()
+        .as_ref()
-            .map(|h| {
+        .map(|h| {
-                if h.is_empty() {
+            if h.is_empty() {
-                    default_homepage.to_string()
+                default_homepage.to_string()
-                } else {
+            } else {
-                    h.to_string()
+                h.to_string()
-                }
+            }
-            })
+        })
-            .unwrap_or(default_homepage.to_string()),
+        .map(|h| Url::parse(&h).ok())
-    )?;
+        .flatten();
     let addr = r.link.as_ref().map(|link| {
         if link.contains('@') {
             link.clone()
@@ -211,16 +193,46 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
             }
         }
     });
-    let html = r.summary.unwrap_or("NO SUMMARY".to_string());
+    let mut body = r.summary.unwrap_or("NO SUMMARY".to_string());
     // TODO: add site specific cleanups. For example:
     // * Grafana does <div class="image-wrapp"><img class="lazyload>"<img src="/media/...>"</img></div>
-    // * Some sites appear to be HTML encoded, unencode them, i.e. imperialviolet
+    // * Some sites appear to be HTML encoded, unencode them, i.e. imperialviolent
-    let html = sanitize_html(&html, "", &link)?;
+    let body_tranformers: Vec<Box<dyn Transformer>> = vec![
+        // TODO: add a map of urls and selectors
+        Box::new(SlurpContents {
+            site_selectors: hashmap![
+                "hackaday.com".to_string() => vec![
+                    Selector::parse("div.entry-featured-image").unwrap(),
+                    Selector::parse("div.entry-content").unwrap()
+                ],
+                "mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
+                "natwelch.com".to_string() => vec![
+                    Selector::parse("article div.prose").unwrap(),
+                ],
+                "slashdot.org".to_string() => vec![
+                    Selector::parse("span.story-byline").unwrap(),
+                    Selector::parse("div.p").unwrap(),
+                ],
+            ],
+        }),
+        Box::new(AddOutlink),
+        Box::new(EscapeHtml),
+        Box::new(InlineStyle),
+        Box::new(SanitizeHtml {
+            cid_prefix: "",
+            base_url: &link,
+        }),
+    ];
+    for t in body_tranformers.iter() {
+        if t.should_run(&link, &body) {
+            body = t.transform(&link, &body).await?;
+        }
+    }
     let body = Body::Html(Html {
-        html,
+        html: body,
         content_tree: "".to_string(),
     });
-    let title = r.title.unwrap_or("NO TITLE".to_string());
+    let title = clean_title(&r.title.unwrap_or("NO TITLE".to_string())).await?;
     let from = Some(Email {
         name: r.name,
         addr: addr.map(|a| a.to_string()),
@@ -248,34 +260,28 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
         }],
     })
 }
-
+pub async fn set_read_status<'ctx>(
-#[derive(Debug)]
+    pool: &PgPool,
-struct Query {
+    query: &str,
-    unread_only: bool,
+    unread: bool,
-    site: Option<String>,
+) -> Result<bool, ServerError> {
-    remainder: Vec<String>,
+    let query: Query = query.parse()?;
+    sqlx::query_file!("sql/set_unread.sql", !unread, query.uid)
+        .execute(pool)
+        .await?;
+    Ok(true)
 }
-
+async fn clean_title(title: &str) -> Result<String, ServerError> {
-impl FromStr for Query {
+    // Make title HTML so html parsers work
-    type Err = Infallible;
+    let mut title = format!("<html>{title}</html>");
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    let title_tranformers: Vec<Box<dyn Transformer>> =
-        let mut unread_only = false;
+        vec![Box::new(EscapeHtml), Box::new(StripHtml)];
-        let mut site = None;
+    // Make title HTML so html parsers work
-        let mut remainder = Vec::new();
+    title = format!("<html>{title}</html>");
-        let site_prefix = format!("tag:{TAG_PREFIX}");
+    for t in title_tranformers.iter() {
-        for word in s.split_whitespace() {
+        if t.should_run(&None, &title) {
-            if word == "is:unread" {
+            title = t.transform(&None, &title).await?;
-                unread_only = true
-            } else if word.starts_with(&site_prefix) {
-                site = Some(word[site_prefix.len()..].to_string())
-            } else {
-                remainder.push(word.to_string());
-            }
         }
-        Ok(Query {
-            unread_only,
-            site,
-            remainder,
-        })
     }
+    Ok(title)
 }

server/src/nm.rs

@@ -5,14 +5,13 @@ use std::{
     time::Instant,
 };
 
-use async_graphql::connection::{self, Connection, Edge};
 use log::{error, info, warn};
 use mailparse::{parse_mail, MailHeader, MailHeaderMap, ParsedMail};
 use memmap::MmapOptions;
 use notmuch::Notmuch;
-use url::Url;
 
 use crate::{
+    compute_offset_limit,
     error::ServerError,
     graphql::{
         Attachment, Body, DispositionType, Email, Header, Html, Message, PlainText, Tag, Thread,
@@ -45,42 +44,29 @@ pub async fn count(nm: &Notmuch, query: &str) -> Result<usize, ServerError> {
 
 pub async fn search(
     nm: &Notmuch,
-    after: Option<String>,
+    after: Option<i32>,
-    before: Option<String>,
+    before: Option<i32>,
     first: Option<i32>,
     last: Option<i32>,
     query: String,
-) -> Result<Connection<usize, ThreadSummary>, async_graphql::Error> {
+) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
-    connection::query(
+    let (offset, mut limit) = compute_offset_limit(after, before, first, last);
-        after,
+    if before.is_none() {
-        before,
+        // When searching forward, the +1 is to see if there are more pages of data available.
-        first,
+        // Searching backwards implies there's more pages forward, because the value represented by
-        last,
+        // `before` is on the next page.
-        |after, before, first, last| async move {
+        limit = limit + 1;
-            let total = nm.count(&query)?;
+    }
-            let (first, last) = if let (None, None) = (first, last) {
+    Ok(nm
-                info!("neither first nor last set, defaulting first to 20");
+        .search(&query, offset as usize, limit as usize)?
-                (Some(20), None)
+        .0
-            } else {
+        .into_iter()
-                (first, last)
+        .enumerate()
-            };
+        .map(|(i, ts)| {
-
+            (
-            let mut start = after.map(|after| after + 1).unwrap_or(0);
+                offset + i as i32,
-            let mut end = before.unwrap_or(total);
+                ThreadSummary {
-            if let Some(first) = first {
+                    thread: format!("thread:{}", ts.thread),
-                end = (start + first).min(end);
-            }
-            if let Some(last) = last {
-                start = if last > end - start { end } else { end - last };
-            }
-
-            let count = end - start;
-            let slice: Vec<ThreadSummary> = nm
-                .search(&query, start, count)?
-                .0
-                .into_iter()
-                .map(|ts| ThreadSummary {
-                    thread: ts.thread,
                     timestamp: ts.timestamp,
                     date_relative: ts.date_relative,
                     matched: ts.matched,
@@ -88,20 +74,10 @@ pub async fn search(
                     authors: ts.authors,
                     subject: ts.subject,
                     tags: ts.tags,
-                })
+                },
-                .collect();
+            )
-
+        })
-            let mut connection = Connection::new(start > 0, end < total);
+        .collect())
-            connection.edges.extend(
-                slice
-                    .into_iter()
-                    .enumerate()
-                    .map(|(idx, item)| Edge::new(start + idx, item)),
-            );
-            Ok::<_, async_graphql::Error>(connection)
-        },
-    )
-    .await
 }
 
 pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
@@ -179,7 +155,7 @@ pub async fn thread(
             .get_first_value("date")
             .and_then(|d| mailparse::dateparse(&d).ok());
         let cid_prefix = shared::urls::cid_prefix(None, &id);
-        let base_url = Url::parse("https://there-should-be-no-relative-urls-in-email").unwrap();
+        let base_url = None;
         let body = match extract_body(&m, &id)? {
             Body::PlainText(PlainText { text, content_tree }) => {
                 let text = if text.len() > MAX_RAW_MESSAGE_SIZE {
@@ -248,7 +224,7 @@ pub async fn thread(
         // TODO(wathiede): parse message and fill out attachments
         let attachments = extract_attachments(&m, &id)?;
         messages.push(Message {
-            id,
+            id: format!("id:{id}"),
             from,
             to,
             cc,
@@ -489,8 +465,10 @@ fn extract_mixed(m: &ParsedMail, part_addr: &mut Vec<String>) -> Result<Body, Se
                     .unwrap_or("".to_string());
                 // Only add inline images, attachments are handled as an attribute of the top level Message and rendered separate client-side.
                 if pcd.disposition == mailparse::DispositionType::Inline {
+                    // TODO: make URL generation more programatic based on what the frontend has
+                    // mapped
                     parts.push(Body::html(format!(
-                        r#"<img src="/view/attachment/{}/{}/{filename}">"#,
+                        r#"<img src="/api/view/attachment/{}/{}/{filename}">"#,
                         part_addr[0],
                         part_addr
                             .iter()
@@ -752,3 +730,16 @@ fn render_content_type_tree(m: &ParsedMail) -> String {
         SKIP_HEADERS.join("\n  ")
     )
 }
+
+pub async fn set_read_status<'ctx>(
+    nm: &Notmuch,
+    query: &str,
+    unread: bool,
+) -> Result<bool, ServerError> {
+    if unread {
+        nm.tag_add("unread", &format!("{query}"))?;
+    } else {
+        nm.tag_remove("unread", &format!("{query}"))?;
+    }
+    Ok(true)
+}

web/Cargo.toml

@@ -36,6 +36,8 @@ wasm-opt = ['-Os']
 [dependencies.web-sys]
 version = "0.3.58"
 features = [
+  "Clipboard",
   "MediaQueryList",
+  "Navigator",
   "Window"
 ]

web/src/state.rs

@@ -291,12 +291,29 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     })
                     .collect(),
             );
+            info!("pager {:#?}", data.search.page_info);
+            let selected_threads = 'context: {
+                if let Context::SearchResult {
+                    results,
+                    selected_threads,
+                    ..
+                } = &model.context
+                {
+                    let old: HashSet<_> = results.iter().map(|n| &n.thread).collect();
+                    let new: HashSet<_> = data.search.nodes.iter().map(|n| &n.thread).collect();
+
+                    if old == new {
+                        break 'context selected_threads.clone();
+                    }
+                }
+                HashSet::new()
+            };
             model.context = Context::SearchResult {
                 query: model.query.clone(),
                 results: data.search.nodes,
                 count: data.count as usize,
                 pager: data.search.page_info,
-                selected_threads: HashSet::new(),
+                selected_threads,
             };
         }
 
@@ -372,7 +389,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             {
                 let threads = selected_threads
                     .iter()
-                    .map(|tid| format!("thread:{tid}"))
+                    .map(|tid| tid.to_string())
                     .collect::<Vec<_>>()
                     .join(" ");
                 orders
@@ -387,7 +404,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             {
                 let threads = selected_threads
                     .iter()
-                    .map(|tid| format!("thread:{tid}"))
+                    .map(|tid| tid.to_string())
                     .collect::<Vec<_>>()
                     .join(" ");
                 orders
@@ -402,7 +419,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             {
                 let threads = selected_threads
                     .iter()
-                    .map(|tid| format!("thread:{tid}"))
+                    .map(|tid| tid.to_string())
                     .collect::<Vec<_>>()
                     .join(" ");
                 orders
@@ -417,7 +434,7 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             {
                 let threads = selected_threads
                     .iter()
-                    .map(|tid| format!("thread:{tid}"))
+                    .map(|tid| tid.to_string())
                     .collect::<Vec<_>>()
                     .join(" ");
                 orders
@@ -452,6 +469,17 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
             }
         }
         Msg::MultiMsg(msgs) => msgs.into_iter().for_each(|msg| update(msg, model, orders)),
+        Msg::CopyToClipboard(text) => {
+            let clipboard = seed::window()
+                .navigator()
+                .clipboard()
+                .expect("couldn't get clipboard");
+            orders.perform_cmd(async move {
+                wasm_bindgen_futures::JsFuture::from(clipboard.write_text(&text))
+                    .await
+                    .expect("failed to copy to clipboard");
+            });
+        }
     }
 }
 // `Model` describes our app state.
@@ -551,4 +579,6 @@ pub enum Msg {
     MessageCollapse(String),
     MessageExpand(String),
     MultiMsg(Vec<Msg>),
+
+    CopyToClipboard(String),
 }

web/src/view/desktop.rs

@@ -3,8 +3,8 @@ use seed_hooks::{state_access::CloneState, topo, use_state};
 
 use crate::{
     api::urls,
-    state::{Context, Model, Msg, Tag},
+    state::{Context, Model, Msg},
-    view::{self, view_header, view_search_results},
+    view::{self, view_header, view_search_results, view_tags},
 };
 
 #[topo::nested]
@@ -33,100 +33,9 @@ pub(super) fn view(model: &Model) -> Node<Msg> {
             show_icon_text,
         ),
     };
-    fn view_tag_li(display_name: &str, indent: usize, t: &Tag, search_unread: bool) -> Node<Msg> {
-        let href = if search_unread {
-            urls::search(&format!("is:unread tag:{}", t.name), 0)
-        } else {
-            urls::search(&format!("tag:{}", t.name), 0)
-        };
-        li![a![
-            attrs! {
-                At::Href => href
-            },
-            (0..indent).map(|_| span![C!["tag-indent"], ""]),
-            i![
-                C!["tag-tag", "fa-solid", "fa-tag"],
-                style! {
-                    //"--fa-primary-color" => t.fg_color,
-                    St::Color => t.bg_color,
-                },
-            ],
-            display_name,
-            IF!(t.unread>0 => format!(" ({})", t.unread))
-        ]]
-    }
-    fn matches(a: &[&str], b: &[&str]) -> usize {
-        std::iter::zip(a.iter(), b.iter())
-            .take_while(|(a, b)| a == b)
-            .count()
-    }
-    fn view_tag_list<'a>(
-        tags: impl Iterator<Item = &'a Tag>,
-        search_unread: bool,
-    ) -> Vec<Node<Msg>> {
-        let mut lis = Vec::new();
-        let mut last = Vec::new();
-        for t in tags {
-            let parts: Vec<_> = t.name.split('/').collect();
-            let mut n = matches(&last, &parts);
-            if n <= parts.len() - 2 && parts.len() > 1 {
-                // Synthesize fake tags for proper indenting.
-                for i in n..parts.len() - 1 {
-                    let display_name = parts[n];
-                    lis.push(view_tag_li(
-                        &display_name,
-                        n,
-                        &Tag {
-                            name: parts[..i + 1].join("/"),
-                            bg_color: "#fff".to_string(),
-                            fg_color: "#000".to_string(),
-                            unread: 0,
-                        },
-                        search_unread,
-                    ));
-                }
-                n = parts.len() - 1;
-            }
-            let display_name = parts[n];
-            lis.push(view_tag_li(&display_name, n, t, search_unread));
-            last = parts;
-        }
-        lis
-    }
-    let unread = model
-        .tags
-        .as_ref()
-        .map(|tags| tags.iter().filter(|t| t.unread > 0).collect())
-        .unwrap_or(Vec::new());
-    let tags_open = use_state(|| false);
-    let force_tags_open = unread.is_empty();
     div![
         C!["main-content"],
-        aside![
+        view_tags(model),
-            C!["tags-menu", "menu"],
-            IF!(!unread.is_empty() => p![C!["menu-label"], "Unread"]),
-            IF!(!unread.is_empty() => ul![C!["menu-list"], view_tag_list(unread.into_iter(),true)]),
-            p![
-                C!["menu-label"],
-                IF!(!force_tags_open =>
-                i![C![
-                    "fa-solid",
-                    if tags_open.get() {
-                        "fa-angle-up"
-                    } else {
-                        "fa-angle-down"
-                    }
-                ]]),
-                " Tags",
-                ev(Ev::Click, move |_| {
-                    tags_open.set(!tags_open.get());
-                })
-            ],
-            ul![
-                C!["menu-list"],
-                IF!(force_tags_open||tags_open.get() => model.tags.as_ref().map(|tags| view_tag_list(tags.iter(),false))),
-            ]
-        ],
         div![
             view_header(&model.query, &model.refreshing_state),
             content,

web/src/view/mobile.rs

@@ -6,7 +6,10 @@ use crate::{
     api::urls,
     graphql::front_page_query::*,
     state::{Context, Model, Msg},
-    view::{self, human_age, pretty_authors, search_toolbar, set_title, tags_chiclet, view_header},
+    view::{
+        self, human_age, pretty_authors, search_toolbar, set_title, tags_chiclet, view_header,
+        view_tags,
+    },
 };
 
 pub(super) fn view(model: &Model) -> Node<Msg> {
@@ -37,6 +40,7 @@ pub(super) fn view(model: &Model) -> Node<Msg> {
         view_header(&model.query, &model.refreshing_state),
         content,
         view_header(&model.query, &model.refreshing_state),
+        view_tags(model),
     ]
 }
 

web/src/view/mod.rs

@@ -6,14 +6,14 @@ use std::{
 use chrono::{DateTime, Datelike, Duration, Local, Utc};
 use human_format::{Formatter, Scales};
 use itertools::Itertools;
-use log::error;
+use log::{error, info};
 use seed::{prelude::*, *};
 use seed_hooks::{state_access::CloneState, topo, use_state};
 
 use crate::{
     api::urls,
     graphql::{front_page_query::*, show_thread_query::*},
-    state::{unread_query, Model, Msg, RefreshingState},
+    state::{unread_query, Model, Msg, RefreshingState, Tag},
 };
 
 mod desktop;
@@ -40,23 +40,11 @@ fn tags_chiclet(tags: &[String], is_mobile: bool) -> impl Iterator<Item = Node<M
         let style = style! {St::BackgroundColor=>hex};
         let classes = C!["tag", IF!(is_mobile => "is-small")];
         let tag = tag.clone();
-        a![
+        a![match tag.as_str() {
-            attrs! {
+            "attachment" => span![classes, style, "📎"],
-                At::Href => urls::search(&format!("tag:{tag}"), 0)
+            "replied" => span![classes, style, i![C!["fa-solid", "fa-reply"]]],
-            },
+            _ => span![classes, style, &tag],
-            match tag.as_str() {
+        },]
-                "attachment" => span![classes, style, "📎"],
-                "replied" => span![classes, style, i![C!["fa-solid", "fa-reply"]]],
-                _ => span![classes, style, &tag],
-            },
-            ev(Ev::Click, move |_| Msg::FrontPageRequest {
-                query: format!("tag:{tag}"),
-                after: None,
-                before: None,
-                first: None,
-                last: None,
-            })
-        ]
     })
 }
 
@@ -73,6 +61,7 @@ fn removable_tags_chiclet<'a>(
             "is-grouped-multiline"
         ],
         tags.iter().map(move |tag| {
+            let thread_id = thread_id.to_string();
             let hex = compute_color(tag);
             let style = style! {St::BackgroundColor=>hex};
             let classes = C!["tag", IF!(is_mobile => "is-small")];
@@ -81,7 +70,6 @@ fn removable_tags_chiclet<'a>(
             };
             let tag = tag.clone();
             let rm_tag = tag.clone();
-            let thread_id = format!("thread:{thread_id}");
             div![
                 C!["control"],
                 div![
@@ -122,14 +110,16 @@ fn pretty_authors(authors: &str) -> impl Iterator<Item = Node<Msg>> + '_ {
             if one_person {
                 return Some(span![
                     attrs! {
-                    At::Title => author.trim()},
+                        At::Title => author.trim()
+                    },
                     author
                 ]);
             }
             author.split_whitespace().nth(0).map(|first| {
                 span![
                     attrs! {
-                    At::Title => author.trim()},
+                        At::Title => author.trim()
+                    },
                     first
                 ]
             })
@@ -146,7 +136,7 @@ fn human_age(timestamp: i64) -> String {
         .with_timezone(&Local);
     let age = now - ts;
     let datetime = if age < Duration::minutes(1) {
-        format!("{} min. ago", age.num_seconds())
+        format!("{} secs. ago", age.num_seconds())
     } else if age < Duration::hours(1) {
         format!("{} min. ago", age.num_minutes())
     } else if ts.date_naive() == now.date_naive() {
@@ -232,10 +222,10 @@ fn view_search_results(
             ],
             td![
                 C!["subject"],
-                tags_chiclet(&tags, false),
-                " ",
                 a![
-                    C!["has-text-light", "text"],
+                    tags_chiclet(&tags, false),
+                    " ",
+                    C!["has-text-light", "text", "subject-link"],
                     attrs! {
                         At::Href => urls::thread(&tid)
                     },
@@ -303,23 +293,11 @@ fn search_toolbar(
     show_bulk_edit: bool,
     show_icon_text: bool,
 ) -> Node<Msg> {
-    let start = pager
-        .start_cursor
-        .as_ref()
-        .map(|i| i.parse().unwrap_or(0))
-        .unwrap_or(0)
-        + 1;
-    let end = pager
-        .end_cursor
-        .as_ref()
-        .map(|i| i.parse().unwrap_or(count))
-        .unwrap_or(count)
-        + 1;
     nav![
         C!["level", "is-mobile"],
-        IF!(show_bulk_edit =>
         div![
             C!["level-left"],
+            IF!(show_bulk_edit =>
             div![
                 C!["level-item"],
                 div![C!["buttons", "has-addons"],
@@ -338,7 +316,8 @@ fn search_toolbar(
                         ev(Ev::Click, |_| Msg::SelectionMarkAsUnread)
                     ]
                 ]
-            ],
+            ]),
+            IF!(show_bulk_edit =>
             div![
                 C!["level-item"],
                 div![C!["buttons", "has-addons"],
@@ -355,8 +334,8 @@ fn search_toolbar(
                         )
                     ],
                 ],
-            ]
+            ])
-        ]),
+        ],
         div![
             C!["level-right"],
             nav![
@@ -381,10 +360,7 @@ fn search_toolbar(
                     ">",
                     IF!(pager.has_next_page => ev(Ev::Click, |_| Msg::NextPage))
                 ],
-                ul![
+                ul![C!["pagination-list"], li![format!("{count} results")],],
-                    C!["pagination-list"],
-                    li![format!("{} - {} of {}", start, end, count)],
-                ],
             ]
         ]
     ]
@@ -442,8 +418,10 @@ fn has_unread(tags: &[String]) -> bool {
 fn render_avatar(avatar: Option<String>, from: &str) -> Node<Msg> {
     let initials: String = from
         .to_lowercase()
+        .trim()
         .split(" ")
         .map(|word| word.chars().next().unwrap())
+        .filter(|c| c.is_alphanumeric())
         // Limit to 2 characters because more characters don't fit in the box
         .take(2)
         .collect();
@@ -516,7 +494,17 @@ fn render_open_header(msg: &ShowThreadQueryThreadMessages) -> Node<Msg> {
                 p![
                     strong![from],
                     br![],
-                    small![from_detail],
+                    small![
+                        &from_detail,
+                        " ",
+                        from_detail.map(|detail| span![
+                            i![C!["far", "fa-clone"]],
+                            ev(Ev::Click, move |e| {
+                                e.stop_propagation();
+                                Msg::CopyToClipboard(detail.to_string())
+                            })
+                        ])
+                    ],
                     table![
                         IF!(!msg.to.is_empty() =>
                         tr![
@@ -526,19 +514,31 @@ fn render_open_header(msg: &ShowThreadQueryThreadMessages) -> Node<Msg> {
                                 msg.to.iter().enumerate().map(|(i, to)|
                                     small![
                                         if i>0 { ", " }else { "" },
-                                        match to {
+                                        {
-                                            ShowThreadQueryThreadMessagesTo {
+                                            let to = match to {
-                                                name: Some(name),
+                                                ShowThreadQueryThreadMessagesTo {
-                                                addr:Some(addr),
+                                                    name: Some(name),
-                                            } => format!("{name} <{addr}>"),
+                                                    addr:Some(addr),
-                                            ShowThreadQueryThreadMessagesTo {
+                                                } => format!("{name} <{addr}>"),
-                                                name: Some(name),
+                                                ShowThreadQueryThreadMessagesTo {
-                                                addr:None
+                                                    name: Some(name),
-                                            } => format!("{name}"),
+                                                    addr:None
-                                            ShowThreadQueryThreadMessagesTo {
+                                                } => format!("{name}"),
-                                                addr: Some(addr), ..
+                                                ShowThreadQueryThreadMessagesTo {
-                                            } => format!("{addr}"),
+                                                    addr: Some(addr), ..
-                                            _ => String::from("UNKNOWN"),
+                                                } => format!("{addr}"),
+                                                _ => String::from("UNKNOWN"),
+                                            };
+                                            span![
+                                                &to, " ",
+                                                span![
+                                                    i![C!["far", "fa-clone"]],
+                                                    ev(Ev::Click, move |e| {
+                                                        e.stop_propagation();
+                                                        Msg::CopyToClipboard(to)
+                                                    })
+                                                ]
+                                            ]
                                         }
 
                                     ])
@@ -551,21 +551,32 @@ fn render_open_header(msg: &ShowThreadQueryThreadMessages) -> Node<Msg> {
                                 msg.cc.iter().enumerate().map(|(i, cc)|
                                     small![
                                         if i>0 { ", " }else { "" },
-                                        match cc {
+                                        {
-                                            ShowThreadQueryThreadMessagesCc {
+                                            let cc = match cc {
-                                                name: Some(name),
+                                                ShowThreadQueryThreadMessagesCc {
-                                                addr:Some(addr),
+                                                    name: Some(name),
-                                            } => format!("{name} <{addr}>"),
+                                                    addr:Some(addr),
-                                            ShowThreadQueryThreadMessagesCc {
+                                                } => format!("{name} <{addr}>"),
-                                                name: Some(name),
+                                                ShowThreadQueryThreadMessagesCc {
-                                                addr:None
+                                                    name: Some(name),
-                                            } => format!("{name}"),
+                                                    addr:None
-                                            ShowThreadQueryThreadMessagesCc {
+                                                } => format!("{name}"),
-                                                addr: Some(addr), ..
+                                                ShowThreadQueryThreadMessagesCc {
-                                            } => format!("<{addr}>"),
+                                                    addr: Some(addr), ..
-                                            _ => String::from("UNKNOWN"),
+                                                } => format!("<{addr}>"),
+                                                _ => String::from("UNKNOWN"),
+                                            };
+                                            span![
+                                                &cc, " ",
+                                                span![
+                                                    i![C!["far", "fa-clone"]],
+                                                    ev(Ev::Click, move |e| {
+                                                        e.stop_propagation();
+                                                        Msg::CopyToClipboard(cc)
+                                                    })
+                                                ]
+                                            ]
                                         }
-
                                     ])
                             ]
                         ]),
@@ -592,7 +603,7 @@ fn render_open_header(msg: &ShowThreadQueryThreadMessages) -> Node<Msg> {
             ],
             ev(Ev::Click, move |e| {
                 e.stop_propagation();
-                Msg::SetUnread(format!("id:{id}"), !is_unread)
+                Msg::SetUnread(id, !is_unread)
             })
         ]
     ]
@@ -664,7 +675,7 @@ fn render_closed_header(msg: &ShowThreadQueryThreadMessages) -> Node<Msg> {
                     ],
                     ev(Ev::Click, move |e| {
                         e.stop_propagation();
-                        Msg::SetUnread(format!("id:{id}"), !is_unread)
+                        Msg::SetUnread(id, !is_unread)
                     })
                 ]
             ]
@@ -808,7 +819,8 @@ fn thread(
     });
     let read_thread_id = thread.thread_id.clone();
     let unread_thread_id = thread.thread_id.clone();
-    let spam_thread_id = thread.thread_id.clone();
+    let spam_add_thread_id = thread.thread_id.clone();
+    let spam_unread_thread_id = thread.thread_id.clone();
     div![
         C!["thread"],
         h3![C!["is-size-5"], subject],
@@ -827,20 +839,14 @@ fn thread(
                         attrs! {At::Title => "Mark as read"},
                         span![C!["icon", "is-small"], i![C!["far", "fa-envelope-open"]]],
                         IF!(show_icon_text=>span!["Read"]),
-                        ev(Ev::Click, move |_| Msg::SetUnread(
+                        ev(Ev::Click, move |_| Msg::SetUnread(read_thread_id, false)),
-                            format!("thread:{read_thread_id}"),
-                            false
-                        )),
                     ],
                     button![
                         C!["button", "mark-unread"],
                         attrs! {At::Title => "Mark as unread"},
                         span![C!["icon", "is-small"], i![C!["far", "fa-envelope"]]],
                         IF!(show_icon_text=>span!["Unread"]),
-                        ev(Ev::Click, move |_| Msg::SetUnread(
+                        ev(Ev::Click, move |_| Msg::SetUnread(unread_thread_id, true)),
-                            format!("thread:{unread_thread_id}"),
-                            true
-                        )),
                     ],
                 ],
             ],
@@ -854,8 +860,8 @@ fn thread(
                         span![C!["icon", "is-small"], i![C!["far", "fa-hand"]]],
                         IF!(show_icon_text=>span!["Spam"]),
                         ev(Ev::Click, move |_| Msg::MultiMsg(vec![
-                            Msg::AddTag(format!("thread:{spam_thread_id}"), "Spam".to_string()),
+                            Msg::AddTag(spam_add_thread_id, "Spam".to_string()),
-                            Msg::SetUnread(format!("thread:{spam_thread_id}"), false)
+                            Msg::SetUnread(spam_unread_thread_id, false)
                         ])),
                     ],
                 ],
@@ -970,3 +976,102 @@ pub fn view(model: &Model) -> Node<Msg> {
         _ => div![C!["desktop"], desktop::view(model)],
     },]
 }
+pub fn view_tags(model: &Model) -> Node<Msg> {
+    fn view_tag_li(display_name: &str, indent: usize, t: &Tag, search_unread: bool) -> Node<Msg> {
+        let href = if search_unread {
+            urls::search(&format!("is:unread tag:{}", t.name), 0)
+        } else {
+            urls::search(&format!("tag:{}", t.name), 0)
+        };
+        li![a![
+            attrs! {
+                At::Href => href
+            },
+            (0..indent).map(|_| span![C!["tag-indent"], ""]),
+            i![
+                C!["tag-tag", "fa-solid", "fa-tag"],
+                style! {
+                    //"--fa-primary-color" => t.fg_color,
+                    St::Color => t.bg_color,
+                },
+            ],
+            display_name,
+            IF!(t.unread>0 => format!(" ({})", t.unread)),
+            ev(Ev::Click, |_| {
+                // Scroll window to the top when searching for a tag.
+                info!("scrolling to the top because you clicked a tag");
+                web_sys::window().unwrap().scroll_to_with_x_and_y(0., 0.);
+            })
+        ]]
+    }
+    fn matches(a: &[&str], b: &[&str]) -> usize {
+        std::iter::zip(a.iter(), b.iter())
+            .take_while(|(a, b)| a == b)
+            .count()
+    }
+    fn view_tag_list<'a>(
+        tags: impl Iterator<Item = &'a Tag>,
+        search_unread: bool,
+    ) -> Vec<Node<Msg>> {
+        let mut lis = Vec::new();
+        let mut last = Vec::new();
+        for t in tags {
+            let parts: Vec<_> = t.name.split('/').collect();
+            let mut n = matches(&last, &parts);
+            if n <= parts.len() - 2 && parts.len() > 1 {
+                // Synthesize fake tags for proper indenting.
+                for i in n..parts.len() - 1 {
+                    let display_name = parts[n];
+                    lis.push(view_tag_li(
+                        &display_name,
+                        n,
+                        &Tag {
+                            name: parts[..i + 1].join("/"),
+                            bg_color: "#fff".to_string(),
+                            fg_color: "#000".to_string(),
+                            unread: 0,
+                        },
+                        search_unread,
+                    ));
+                }
+                n = parts.len() - 1;
+            }
+            let display_name = parts[n];
+            lis.push(view_tag_li(&display_name, n, t, search_unread));
+            last = parts;
+        }
+        lis
+    }
+    let unread = model
+        .tags
+        .as_ref()
+        .map(|tags| tags.iter().filter(|t| t.unread > 0).collect())
+        .unwrap_or(Vec::new());
+    let tags_open = use_state(|| false);
+    let force_tags_open = unread.is_empty();
+    aside![
+        C!["tags-menu", "menu"],
+        IF!(!unread.is_empty() => p![C!["menu-label"], "Unread"]),
+        IF!(!unread.is_empty() => ul![C!["menu-list"], view_tag_list(unread.into_iter(),true)]),
+        p![
+            C!["menu-label"],
+            IF!(!force_tags_open =>
+            i![C![
+                "fa-solid",
+                if tags_open.get() {
+                    "fa-angle-up"
+                } else {
+                    "fa-angle-down"
+                }
+            ]]),
+            " Tags",
+            ev(Ev::Click, move |_| {
+                tags_open.set(!tags_open.get());
+            })
+        ],
+        ul![
+            C!["menu-list"],
+            IF!(force_tags_open||tags_open.get() => model.tags.as_ref().map(|tags| view_tag_list(tags.iter(),false))),
+        ]
+    ]
+}

web/src/view/tablet.rs

@@ -2,7 +2,7 @@ use seed::{prelude::*, *};
 
 use crate::{
     state::{Context, Model, Msg},
-    view::{self, view_header, view_search_results},
+    view::{self, view_header, view_search_results, view_tags},
 };
 
 pub(super) fn view(model: &Model) -> Node<Msg> {
@@ -36,6 +36,7 @@ pub(super) fn view(model: &Model) -> Node<Msg> {
             view_header(&model.query, &model.refreshing_state),
             content,
             view_header(&model.query, &model.refreshing_state),
+            view_tags(model),
         ]
     ]
 }

web/static/style.css

@@ -1,3 +1,44 @@
+:root {
+  --active-brightness: 0.85;
+  --border-radius: 5px;
+  --box-shadow: 2px 2px 10px;
+  --color-accent: #118bee15;
+  --color-bg: #fff;
+  --color-bg-secondary: #e9e9e9;
+  --color-link: #118bee;
+  --color-secondary: #920de9;
+  --color-secondary-accent: #920de90b;
+  --color-shadow: #f4f4f4;
+  --color-table: #118bee;
+  --color-text: #000;
+  --color-text-secondary: #999;
+  --color-scrollbar: #cacae8;
+  --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
+  --hover-brightness: 1.2;
+  --justify-important: center;
+  --justify-normal: left;
+  --line-height: 1.5;
+  --width-card: 285px;
+  --width-card-medium: 460px;
+  --width-card-wide: 800px;
+  --width-content: 1080px;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root[color-mode="user"] {
+    --color-accent: #0097fc4f;
+    --color-bg: #333;
+    --color-bg-secondary: #555;
+    --color-link: #0097fc;
+    --color-secondary: #e20de9;
+    --color-secondary-accent: #e20de94f;
+    --color-shadow: #bbbbbb20;
+    --color-table: #0097fc;
+    --color-text: #f7f7f7;
+    --color-text-secondary: #aaa;
+  }
+}
+
 .message {
   display: inline-block;
   padding: 0.5em;
@@ -65,8 +106,9 @@
 }
 
 .view-part-text-plain {
-  padding: 0.5em;
+  font-family: monospace;
   overflow-wrap: break-word;
+  padding: 0.5em;
   white-space: pre-wrap;
   word-break: break-word;
   word-wrap: break-word;
@@ -167,6 +209,10 @@ input::placeholder,
   padding: 1em;
 }
 
+.search-results>nav {
+  margin: 1.25rem;
+}
+
 .tablet .thread h3,
 .mobile .thread h3 {
   overflow-wrap: break-word;
@@ -189,8 +235,6 @@ input::placeholder,
   width: 100%;
 }
 
-.search-results .row .checkbox {}
-
 .search-results .row .summary {
   min-width: 0;
   width: 100%;
@@ -202,16 +246,13 @@ input::placeholder,
   white-space: nowrap;
 }
 
-.search-results td.subject {}
-
-.search-results .subject .tag {}
-
 .search-results .subject .text {
-  padding-left: 0.5rem;
+  display: inline-block;
-  width: 100%;
   overflow: hidden;
+  padding-left: 0.5rem;
   text-overflow: ellipsis;
   white-space: nowrap;
+  width: 100%;
 }
 
 .search-results .row .from {