server: strip style & script tags, also handle some retryable errors on slurp

server/src/lib.rs

@@ -7,7 +7,13 @@ pub mod nm;
 #[cfg(feature = "tantivy")]
 pub mod tantivy;
 
-use std::{collections::HashMap, convert::Infallible, fmt, str::FromStr, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    convert::Infallible,
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
 
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
@@ -20,6 +26,8 @@ use lol_html::{
 };
 use maplit::{hashmap, hashset};
 use regex::Regex;
+use reqwest::StatusCode;
+use rocket::response::status;
 use scraper::{Html, Selector};
 use sqlx::types::time::PrimitiveDateTime;
 use thiserror::Error;
@@ -58,6 +66,8 @@ pub enum TransformError {
     ReqwestError(#[from] reqwest::Error),
     #[error("failed to parse HTML: {0}")]
     HtmlParsingError(String),
+    #[error("got a retryable error code {0} for {1}")]
+    RetryableHttpStatusError(StatusCode, String),
 }
 
 struct SanitizeHtml<'a> {
@@ -88,18 +98,37 @@ struct StripHtml;
 
 #[async_trait]
 impl Transformer for StripHtml {
-    fn should_run(&self, _: &Option<Url>, html: &str) -> bool {
+    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("StripHtml should_run {link:?} {}", html.contains("<"));
         // Lame test
         html.contains("<")
     }
-    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
+    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("StripHtml {link:?}");
         let mut text = String::new();
+        let element_content_handlers = vec![
+            element!("style", |el| {
+                el.remove();
+                Ok(())
+            }),
+            element!("script", |el| {
+                el.remove();
+                Ok(())
+            }),
+        ];
+        let html = rewrite_str(
+            html,
+            RewriteStrSettings {
+                element_content_handlers,
+                ..RewriteStrSettings::default()
+            },
+        )?;
         let element_content_handlers = vec![text!("*", |t| {
             text += t.as_str();
             Ok(())
         })];
         let _ = rewrite_str(
-            html,
+            &html,
             RewriteStrSettings {
                 element_content_handlers,
                 ..RewriteStrSettings::default()
@@ -272,6 +301,7 @@ impl<'c> SlurpContents<'c> {
 #[async_trait]
 impl<'c> Transformer for SlurpContents<'c> {
     fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+        debug!("SlurpContents should_run {link:?}");
         let mut will_slurp = false;
         if let Some(link) = link {
             will_slurp = self.get_selectors(link).is_some();
@@ -282,6 +312,15 @@ impl<'c> Transformer for SlurpContents<'c> {
         will_slurp
     }
     async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+        debug!("SlurpContents {link:?}");
+        let retryable_status: HashSet<StatusCode> = vec![
+            StatusCode::UNAUTHORIZED,
+            StatusCode::FORBIDDEN,
+            StatusCode::REQUEST_TIMEOUT,
+            StatusCode::TOO_MANY_REQUESTS,
+        ]
+        .into_iter()
+        .collect();
         if let Some(test_link) = link {
             // If SlurpContents is configured for inline CSS, but no
             // configuration found for this site, use the local InlineStyle
@@ -301,11 +340,18 @@ impl<'c> Transformer for SlurpContents<'c> {
         let body = if let Some(body) = cacher.get(link.as_str()) {
             String::from_utf8_lossy(&body).to_string()
         } else {
-            let body = reqwest::get(link.as_str())
-                .await?
-                .error_for_status()?
-                .text()
-                .await?;
+            let resp = reqwest::get(link.as_str()).await?;
+            let status = resp.status();
+            if status.is_server_error() || retryable_status.contains(&status) {
+                return Err(TransformError::RetryableHttpStatusError(
+                    status,
+                    link.to_string(),
+                ));
+            }
+            if !status.is_success() {
+                return Ok(html.to_string());
+            }
+            let body = resp.text().await?;
             cacher.set(link.as_str(), body.as_bytes());
             body
         };