Bumping version to 0.0.128

Cargo.lock

@@ -2910,7 +2910,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "letterbox"
-version = "0.0.127"
+version = "0.0.128"
 dependencies = [
  "build-info",
  "build-info-build",
@@ -2936,7 +2936,7 @@ dependencies = [
 
 [[package]]
 name = "letterbox-server"
-version = "0.0.127"
+version = "0.0.128"
 dependencies = [
  "ammonia",
  "anyhow",
@@ -3457,7 +3457,7 @@ dependencies = [
 
 [[package]]
 name = "notmuch"
-version = "0.0.127"
+version = "0.0.128"
 dependencies = [
  "itertools 0.10.5",
  "log",
@@ -4252,7 +4252,7 @@ dependencies = [
 
 [[package]]
 name = "procmail2notmuch"
-version = "0.0.127"
+version = "0.0.128"
 dependencies = [
  "anyhow",
 ]
@@ -5331,7 +5331,7 @@ dependencies = [
 
 [[package]]
 name = "shared"
-version = "0.0.127"
+version = "0.0.128"
 dependencies = [
  "build-info",
  "notmuch",

notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "notmuch"
-version = "0.0.127"
+version = "0.0.128"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

procmail2notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "procmail2notmuch"
-version = "0.0.127"
+version = "0.0.128"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

server/.sqlx/query-118e453e59594487fd01873c07d3dc9c8069187d1bcc6d30f6b4940fe694f0eb.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE search_summary IS NULL\nORDER BY date DESC\nLIMIT 100;\n",
+  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n    post AS p\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE\n    search_summary IS NULL\n    -- TODO remove\n    AND link ~ '^<'\nORDER BY date DESC\nLIMIT 100;\n",
   "describe": {
     "columns": [
       {
@@ -28,5 +28,5 @@
       true
     ]
   },
-  "hash": "e90019c1e96c20318b23ef5671cf04e48b42477178d068ced4dba7bb6d5896d5"
+  "hash": "118e453e59594487fd01873c07d3dc9c8069187d1bcc6d30f6b4940fe694f0eb"
 }

server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "letterbox-server"
-version = "0.0.127"
+version = "0.0.128"
 edition = "2021"
 default-run = "letterbox-server"
 

server/src/lib.rs

@@ -7,7 +7,13 @@ pub mod nm;
 #[cfg(feature = "tantivy")]
 pub mod tantivy;
 
-use std::{collections::HashMap, convert::Infallible, fmt, str::FromStr, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    convert::Infallible,
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
 
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
@@ -20,6 +26,8 @@ use lol_html::{
 };
 use maplit::{hashmap, hashset};
 use regex::Regex;
+use reqwest::StatusCode;
+use rocket::response::status;
 use scraper::{Html, Selector};
 use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
@@ -58,6 +66,8 @@ pub enum TransformError {
     ReqwestError(#[from] reqwest::Error),
     #[error("failed to parse HTML: {0}")]
     HtmlParsingError(String),
+    #[error("got a retryable error code {0} for {1}")]
+    RetryableHttpStatusError(StatusCode, String),
 }
 
 struct SanitizeHtml<'a> {
@@ -88,18 +98,37 @@ struct StripHtml;
 
 #[async_trait]
 impl Transformer for StripHtml {
-    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("StripHtml should_run {link:?} {}", html.contains("<"));
         // Lame test
         html.contains("<")
     }
-    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("StripHtml {link:?}");
         let mut text = String::new();
+        let element_content_handlers = vec![
+            element!("style", |el| {
+                el.remove();
+                Ok(())
+            }),
+            element!("script", |el| {
+                el.remove();
+                Ok(())
+            }),
+        ];
+        let html = rewrite_str(
+            html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
         let element_content_handlers = vec![text!("*", |t| {
             text += t.as_str();
             Ok(())
         })];
         let _ = rewrite_str(
-            html,
+            &html,
             RewriteStrSettings {
                 element_content_handlers,
                 ..RewriteStrSettings::default()
@@ -272,6 +301,7 @@ impl<'c> SlurpContents<'c> {
 #[async_trait]
 impl<'c> Transformer for SlurpContents<'c> {
     fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("SlurpContents should_run {link:?}");
         let mut will_slurp = false;
         if let Some(link) = link {
             will_slurp = self.get_selectors(link).is_some();
@@ -282,6 +312,15 @@ impl<'c> Transformer for SlurpContents<'c> {
         will_slurp
     }
     async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("SlurpContents {link:?}");
+        let retryable_status: HashSet<StatusCode> = vec![
+            StatusCode::UNAUTHORIZED,
+            StatusCode::FORBIDDEN,
+            StatusCode::REQUEST_TIMEOUT,
+            StatusCode::TOO_MANY_REQUESTS,
+        ]
+        .into_iter()
+        .collect();
         if let Some(test_link) = link {
             // If SlurpContents is configured for inline CSS, but no
             // configuration found for this site, use the local InlineStyle
@@ -301,11 +340,18 @@ impl<'c> Transformer for SlurpContents<'c> {
         let body = if let Some(body) = cacher.get(link.as_str()) {
             String::from_utf8_lossy(&body).to_string()
         } else {
-            let body = reqwest::get(link.as_str())
+            let resp = reqwest::get(link.as_str()).await?;
-                .await?
+            let status = resp.status();
-                .error_for_status()?
+            if status.is_server_error() || retryable_status.contains(&status) {
-                .text()
+                return Err(TransformError::RetryableHttpStatusError(
-                .await?;
+                    status,
+                    link.to_string(),
+                ));
+            }
+            if !status.is_success() {
+                return Ok(html.to_string());
+            }
+            let body = resp.text().await?;
             cacher.set(link.as_str(), body.as_bytes());
             body
         };

server/src/newsreader.rs

@@ -258,27 +258,28 @@ pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<b
     async fn update_search_summary(
         pool: &PgPool,
         cacher: &FilesystemCacher,
-        link: Url,
+        link: String,
         body: String,
         id: i32,
     ) -> Result<(), ServerError> {
-        let body_transformers: Vec<Box<dyn Transformer>> = vec![
+        let slurp_contents = SlurpContents {
-            Box::new(SlurpContents {
             cacher,
             inline_css: true,
             site_selectors: slurp_contents_selectors(),
-            }),
+        };
-            Box::new(StripHtml),
+        let strip_html = StripHtml;
-        ];
 
         info!("adding {link} to search index");
         let mut body = body;
+        if let Ok(link) = Url::parse(&link) {
             let link = Some(link);
-        for t in body_transformers.iter() {
+            if slurp_contents.should_run(&link, &body) {
-            if t.should_run(&link, &body) {
+                body = slurp_contents.transform(&link, &body).await?;
-                body = t.transform(&link, &body).await?;
             }
+        } else {
+            error!("failed to parse link: {}", link);
         }
+        body = strip_html.transform(&None, &body).await?;
         sqlx::query!(
             "UPDATE post SET search_summary = $1 WHERE id = $2",
             body,
@@ -294,16 +295,12 @@ pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<b
         .await?
         .into_iter()
         .filter_map(|r| {
-            let Ok(link) = Url::parse(&r.link) else {
-                error!("failed to parse link: {}", r.link);
-                return None;
-            };
             let Some(body) = r.clean_summary else {
                 error!("clean_summary missing for {}", r.link);
                 return None;
             };
             let id = r.id;
-            Some(update_search_summary(pool, cacher, link, body, id))
+            Some(update_search_summary(pool, cacher, r.link, body, id))
         })
         .collect();
 

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shared"
-version = "0.0.127"
+version = "0.0.128"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

web/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-version = "0.0.127"
+version = "0.0.128"
 name = "letterbox"
 repository = "https://github.com/seed-rs/seed-quickstart"
 authors = ["Bill Thiede <git@xinu.tv>"]