Bumping version to 0.0.122

Cargo.lock

@@ -2910,7 +2910,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "letterbox"
-version = "0.0.121"
+version = "0.0.122"
 dependencies = [
  "build-info",
  "build-info-build",
@@ -2936,7 +2936,7 @@ dependencies = [
 
 [[package]]
 name = "letterbox-server"
-version = "0.0.121"
+version = "0.0.122"
 dependencies = [
  "ammonia",
  "anyhow",
@@ -2949,6 +2949,7 @@ dependencies = [
  "chrono",
  "clap",
  "css-inline",
+ "futures 0.3.31",
  "html-escape",
  "linkify",
  "log",
@@ -3456,7 +3457,7 @@ dependencies = [
 
 [[package]]
 name = "notmuch"
-version = "0.0.121"
+version = "0.0.122"
 dependencies = [
  "itertools 0.10.5",
  "log",
@@ -4251,7 +4252,7 @@ dependencies = [
 
 [[package]]
 name = "procmail2notmuch"
-version = "0.0.121"
+version = "0.0.122"
 dependencies = [
  "anyhow",
 ]
@@ -5330,7 +5331,7 @@ dependencies = [
 
 [[package]]
 name = "shared"
-version = "0.0.121"
+version = "0.0.122"
 dependencies = [
  "build-info",
  "notmuch",

notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "notmuch"
-version = "0.0.121"
+version = "0.0.122"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

procmail2notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "procmail2notmuch"
-version = "0.0.121"
+version = "0.0.122"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

server/.sqlx/query-26e150d5d294f2eb264bbae21766295c4e4e5fe1e076dd7d6bc168e5fb49dc84.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n    post AS p TABLESAMPLE SYSTEM (.1)\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE search_summary IS NULL;\n",
+  "query": "SELECT\n    p.id,\n    link,\n    clean_summary\nFROM\n-- Remoe tablesample when db sufficiently indexed\n    post AS p TABLESAMPLE SYSTEM (.1)\nINNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts\nWHERE search_summary IS NULL;\n",
   "describe": {
     "columns": [
       {
@@ -28,5 +28,5 @@
       true
     ]
   },
-  "hash": "078322a8e99faa6b91e3890501f6763e8787b181d5fd070267180b0d5180ca36"
+  "hash": "26e150d5d294f2eb264bbae21766295c4e4e5fe1e076dd7d6bc168e5fb49dc84"
 }

server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "letterbox-server"
-version = "0.0.121"
+version = "0.0.122"
 edition = "2021"
 default-run = "letterbox-server"
 
@@ -17,6 +17,7 @@ cacher = { version = "0.1.0", registry = "xinu" }
 chrono = "0.4.39"
 clap = { version = "4.5.23", features = ["derive"] }
 css-inline = "0.13.0"
+futures = "0.3.31"
 html-escape = "0.2.13"
 linkify = "0.10.0"
 log = "0.4.17"

server/sql/need-search-summary.sql

@@ -3,6 +3,7 @@ SELECT
     link,
     clean_summary
 FROM
+-- Remoe tablesample when db sufficiently indexed
     post AS p TABLESAMPLE SYSTEM (.1)
 INNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
 WHERE search_summary IS NULL;

server/src/newsreader.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use cacher::FilesystemCacher;
+use futures::{stream::FuturesUnordered, StreamExt};
 use log::info;
 use maplit::hashmap;
 use scraper::Selector;
@@ -254,23 +255,25 @@ pub async fn set_read_status<'ctx>(
 }
 #[instrument(name = "newsreader::refresh", skip_all)]
 pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<bool, ServerError> {
-    let body_transformers: Vec<Box<dyn Transformer>> = vec![
+    async fn update_search_summary(
-        Box::new(SlurpContents {
+        pool: &PgPool,
-            cacher,
+        cacher: &FilesystemCacher,
-            inline_css: true,
+        link: Url,
-            site_selectors: slurp_contents_selectors(),
+        body: String,
-        }),
+        id: i32,
-        Box::new(StripHtml),
+    ) -> Result<(), ServerError> {
-    ];
+        let body_transformers: Vec<Box<dyn Transformer>> = vec![
+            Box::new(SlurpContents {
+                cacher,
+                inline_css: true,
+                site_selectors: slurp_contents_selectors(),
+            }),
+            Box::new(StripHtml),
+        ];
 
-    let rows = sqlx::query_file!("sql/need-search-summary.sql",)
-        .fetch_all(pool)
-        .await?;
-    for r in rows {
-        let link = Url::parse(&r.link)?;
         info!("adding {link} to search index");
+        let mut body = body;
         let link = Some(link);
-        let mut body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
         for t in body_transformers.iter() {
             if t.should_run(&link, &body) {
                 body = t.transform(&link, &body).await?;
@@ -279,10 +282,35 @@ pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<b
         sqlx::query!(
             "UPDATE post SET search_summary = $1 WHERE id = $2",
             body,
-            r.id
+            id
         )
         .execute(pool)
         .await?;
+        Ok(())
+    }
+
+    let mut unordered: FuturesUnordered<_> = sqlx::query_file!("sql/need-search-summary.sql",)
+        .fetch_all(pool)
+        .await?
+        .into_iter()
+        .map(|r| {
+            let link = Url::parse(&r.link).expect("failed to parse link");
+            let body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
+            let id = r.id;
+            update_search_summary(pool, cacher, link, body, id)
+        })
+        .collect();
+
+    while let Some(res) = unordered.next().await {
+        //let res = res;
+        match res {
+            Ok(()) => {}
+            Err(err) => {
+                info!("failed refresh {err:?}");
+                // TODO:
+                //fd.error = Some(err);
+            }
+        };
     }
     Ok(true)
 }

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shared"
-version = "0.0.121"
+version = "0.0.122"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

web/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-version = "0.0.121"
+version = "0.0.122"
 name = "letterbox"
 repository = "https://github.com/seed-rs/seed-quickstart"
 authors = ["Bill Thiede <git@xinu.tv>"]