server: improve cloudflare and grafana image and iframe rendering

server/src/bin/server.rs

@@ -1,3 +1,6 @@
+// Rocket generates a lot of warnings for handlers
+// TODO: figure out why
+#![allow(unreachable_patterns)]
 #[macro_use]
 extern crate rocket;
 use std::{error::Error, io::Cursor, str::FromStr};

server/src/lib.rs

@@ -14,19 +14,16 @@ use lol_html::{
     RewriteStrSettings,
 };
 use maplit::{hashmap, hashset};
-use scraper::{error::SelectorErrorKind, Html, Selector};
+use scraper::{Html, Selector};
 use thiserror::Error;
 use url::Url;
 
-use crate::newsreader::{
-    extract_thread_id, is_newsreader_search, is_newsreader_thread, make_news_tag,
-};
-const NON_EXISTENT_SITE_NAME: &'static str = "NO-SUCH-SITE";
+use crate::newsreader::{extract_thread_id, is_newsreader_thread};
 
 // TODO: figure out how to use Cow
 #[async_trait]
 trait Transformer: Send + Sync {
-    fn should_run(&self, addr: &Option<Url>, _html: &str) -> bool {
+    fn should_run(&self, _addr: &Option<Url>, _html: &str) -> bool {
         true
     }
     // TODO: should html be something like `html_escape` uses:
@@ -132,24 +129,44 @@ impl Transformer for InlineStyle {
     }
 }
 
-/// Frame images will extract any alt or title tags on images and place them as labels below said
-/// image.
+/// Process images will extract any alt or title tags on images and place them as labels below said
+/// image. It also handles data-src and data-cfsrc attributes
 struct FrameImages;
 
 #[async_trait]
 impl Transformer for FrameImages {
-    async fn transform(&self, link: &Option<Url>, html: &str) -> Result<String, TransformError> {
+    async fn transform(&self, _: &Option<Url>, html: &str) -> Result<String, TransformError> {
         Ok(rewrite_str(
             html,
             RewriteStrSettings {
-                element_content_handlers: vec![element!("img[alt], img[title]", |el| {
+                element_content_handlers: vec![
+                    element!("img[data-src]", |el| {
+                        info!("found image with data-src {el:?}");
+                        let src = el
+                            .get_attribute("data-src")
+                            .unwrap_or("https://placehold.co/600x400".to_string());
+                        el.set_attribute("src", &src)?;
+
+                        Ok(())
+                    }),
+                    element!("img[data-cfsrc]", |el| {
+                        info!("found image with data-cfsrc {el:?}");
+                        let src = el
+                            .get_attribute("data-cfsrc")
+                            .unwrap_or("https://placehold.co/600x400".to_string());
+                        el.set_attribute("src", &src)?;
+
+                        Ok(())
+                    }),
+                    element!("img[alt], img[title]", |el| {
                         info!("found image with alt or title {el:?}");
                         let src = el
                             .get_attribute("src")
                             .unwrap_or("https://placehold.co/600x400".to_string());
                         let alt = el.get_attribute("alt");
                         let title = el.get_attribute("title");
-                    let mut frags = vec!["<figure>".to_string(), format!(r#"<img src="{src}">"#)];
+                        let mut frags =
+                            vec!["<figure>".to_string(), format!(r#"<img src="{src}">"#)];
                         alt.map(|t| {
                             if !t.is_empty() {
                                 frags.push(format!("<figcaption>Alt: {t}</figcaption>"))
@@ -164,7 +181,8 @@ impl Transformer for FrameImages {
                         el.replace(&frags.join("\n"), ContentType::Html);
 
                         Ok(())
-                })],
+                    }),
+                ],
                 ..RewriteStrSettings::default()
             },
         )?)
@@ -213,7 +231,7 @@ impl SlurpContents {
 
 #[async_trait]
 impl Transformer for SlurpContents {
-    fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
+    fn should_run(&self, link: &Option<Url>, _: &str) -> bool {
         if let Some(link) = link {
             return self.get_selectors(link).is_some();
         }
@@ -275,6 +293,24 @@ pub fn sanitize_html(
     cid_prefix: &str,
     base_url: &Option<Url>,
 ) -> Result<String, TransformError> {
+    let inline_opts = InlineOptions {
+        inline_style_tags: true,
+        keep_style_tags: false,
+        keep_link_tags: false,
+        base_url: None,
+        load_remote_stylesheets: false,
+        extra_css: None,
+        preallocate_node_capacity: 32,
+        ..InlineOptions::default()
+    };
+
+    let html = match CSSInliner::new(inline_opts).inline(&html) {
+        Ok(inlined_html) => inlined_html,
+        Err(err) => {
+            error!("failed to inline CSS: {err}");
+            html.to_string()
+        }
+    };
     let mut element_content_handlers = vec![
         // Open links in new tab
         element!("a[href]", |el| {
@@ -325,25 +361,13 @@ pub fn sanitize_html(
             }),
         ]);
     }
-
-    let inline_opts = InlineOptions {
-        inline_style_tags: true,
-        keep_style_tags: false,
-        keep_link_tags: false,
-        base_url: None,
-        load_remote_stylesheets: false,
-        extra_css: None,
-        preallocate_node_capacity: 32,
-        ..InlineOptions::default()
-    };
-
-    let inlined_html = match CSSInliner::new(inline_opts).inline(&html) {
-        Ok(inlined_html) => inlined_html,
-        Err(err) => {
-            error!("failed to inline CSS: {err}");
-            html.to_string()
-        }
-    };
+    let html = rewrite_str(
+        &html,
+        RewriteStrSettings {
+            element_content_handlers,
+            ..RewriteStrSettings::default()
+        },
+    )?;
     // Default's don't allow style, but we want to preserve that.
     // TODO: remove 'class' if rendering mails moves to a two phase process where abstract message
     // types are collected, santized, and then grouped together as one big HTML doc
@@ -391,6 +415,7 @@ pub fn sanitize_html(
         "hgroup",
         "hr",
         "i",
+        "iframe", // wathiede
         "img",
         "ins",
         "kbd",
@@ -399,6 +424,7 @@ pub fn sanitize_html(
         "map",
         "mark",
         "nav",
+        "noscript", // wathiede
         "ol",
         "p",
         "pre",
@@ -452,6 +478,9 @@ pub fn sanitize_html(
         "hr" => hashset![
             "align", "size", "width"
         ],
+        "iframe" => hashset![
+            "src", "allow", "allowfullscreen"
+        ],
         "img" => hashset![
             "align", "alt", "height", "src", "width"
         ],
@@ -487,21 +516,14 @@ pub fn sanitize_html(
         ],
     ];
 
-    let rewritten_html = rewrite_str(
-        &inlined_html,
-        RewriteStrSettings {
-            element_content_handlers,
-            ..RewriteStrSettings::default()
-        },
-    )?;
-    let clean_html = ammonia::Builder::default()
+    let html = ammonia::Builder::default()
         .tags(tags)
         .tag_attributes(tag_attributes)
         .generic_attributes(attributes)
-        .clean(&rewritten_html)
+        .clean(&html)
         .to_string();
 
-    Ok(clean_html)
+    Ok(html)
 }
 
 fn compute_offset_limit(
@@ -580,7 +602,6 @@ impl FromStr for Query {
         let mut tag = None;
         let mut uid = None;
         let mut remainder = Vec::new();
-        let site_prefix = make_news_tag("");
         let mut is_notmuch = false;
         let mut is_newsreader = false;
         for word in s.split_whitespace() {

server/src/newsreader.rs

@@ -14,7 +14,7 @@ const THREAD_PREFIX: &'static str = "news:";
 use crate::{
     compute_offset_limit,
     error::ServerError,
-    graphql::{Body, Email, Html, Message, NewsPost, Tag, Thread, ThreadSummary},
+    graphql::{NewsPost, Tag, Thread, ThreadSummary},
     AddOutlink, EscapeHtml, FrameImages, InlineStyle, SanitizeHtml, SlurpContents, StripHtml,
     Transformer,
 };
@@ -176,6 +176,10 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
     let body_tranformers: Vec<Box<dyn Transformer>> = vec![
         Box::new(SlurpContents {
             site_selectors: hashmap![
+                "blog.cloudflare.com".to_string() => vec![
+                    Selector::parse(".author-lists").unwrap(),
+                    Selector::parse(".post-full-content").unwrap()
+                ],
                 "hackaday.com".to_string() => vec![
                     Selector::parse("div.entry-featured-image").unwrap(),
                     Selector::parse("div.entry-content").unwrap()

web/src/state.rs

@@ -307,7 +307,6 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .map(|t| Tag {
                         name: t.name,
                         bg_color: t.bg_color,
-                        fg_color: t.fg_color,
                         unread: t.unread,
                     })
                     .collect(),
@@ -357,18 +356,13 @@ pub fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
                     .map(|t| Tag {
                         name: t.name,
                         bg_color: t.bg_color,
-                        fg_color: t.fg_color,
                         unread: t.unread,
                     })
                     .collect(),
             );
             match &data.thread {
                 graphql::show_thread_query::ShowThreadQueryThread::EmailThread(
-                    ShowThreadQueryThreadOnEmailThread {
-                        thread_id,
-                        subject,
-                        messages,
-                    },
+                    ShowThreadQueryThreadOnEmailThread { messages, .. },
                 ) => {
                     let mut open_messages: HashSet<_> = messages
                         .iter()
@@ -555,7 +549,6 @@ pub enum Context {
 pub struct Tag {
     pub name: String,
     pub bg_color: String,
-    pub fg_color: String,
     pub unread: i64,
 }
 
@@ -604,6 +597,8 @@ pub enum Msg {
     SelectionSetNone,
     SelectionSetAll,
     SelectionAddTag(String),
+    #[allow(dead_code)]
+    // TODO
     SelectionRemoveTag(String),
     SelectionMarkAsRead,
     SelectionMarkAsUnread,

web/src/view/desktop.rs

@@ -1,8 +1,7 @@
 use seed::{prelude::*, *};
-use seed_hooks::{state_access::CloneState, topo, use_state};
+use seed_hooks::topo;
 
 use crate::{
-    api::urls,
     graphql::show_thread_query::*,
     state::{Context, Model, Msg},
     view::{self, reading_progress, view_header, view_search_results, view_tags},

web/src/view/mod.rs

@@ -1028,7 +1028,6 @@ pub fn view_tags(model: &Model) -> Node<Msg> {
                         &Tag {
                             name: parts[..i + 1].join("/"),
                             bg_color: "#fff".to_string(),
-                            fg_color: "#000".to_string(),
                             unread: 0,
                         },
                         search_unread,

web/src/view/tablet.rs

@@ -1,4 +1,3 @@
-use log::info;
 use seed::{prelude::*, *};
 
 use crate::{