wathiede/letterbox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

server: use fetched contents of news for search index

Browse Source
This commit is contained in:
Bill Thiede 2025-01-29 14:08:20 -08:00
parent c7aa32b922
commit 12c8e0e33b
12 changed files with 168 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -2958,6 +2958,7 @@ dependencies = [
"memmap",
"notmuch",
"opentelemetry",
"regex",
"reqwest 0.12.9",
"rocket",
"rocket_cors",

1
server/Cargo.toml
View File

@ -26,6 +26,7 @@ maplit = "1.0.2"
memmap = "0.7.0"
notmuch = { path = "../notmuch" }
opentelemetry = "0.27.1"
regex = "1.11.1"
reqwest = { version = "0.12.7", features = ["blocking"] }
rocket = { version = "0.5.0-rc.2", features = ["json"] }
rocket_cors = "0.6.0"

15
server/migrations/20250128234348_add-search-summary.down.sql Normal file
View File

@ -0,0 +1,15 @@
-- Add down migration script here
BEGIN;
DROP INDEX IF EXISTS post_search_summary_idx;
ALTER TABLE post DROP search_summary;
-- CREATE INDEX post_summary_idx ON post USING gin (to_tsvector(
-- 'english',
-- regexp_replace(
-- regexp_replace(summary, '<[^>]+>', ' ', 'g'),
-- '\s+',
-- ' ',
-- 'g'
-- )
-- ));
COMMIT;

14
server/migrations/20250128234348_add-search-summary.up.sql Normal file
View File

@ -0,0 +1,14 @@
-- Add up migration script here
BEGIN;
DROP INDEX IF EXISTS post_summary_idx;
ALTER TABLE post ADD search_summary TEXT;
CREATE INDEX post_search_summary_idx ON post USING gin (
to_tsvector('english', search_summary)
);
UPDATE post SET search_summary = regexp_replace(
regexp_replace(summary, '<[^>]+>', ' ', 'g'),
'\s+',
' ',
'g'
);
COMMIT;

10
server/sql/count.sql
View File

@ -1,10 +1,9 @@
SELECT
COUNT(*) count
SELECT COUNT(*) AS count
FROM
post
WHERE
(
$1 :: text IS NULL
$1::text IS NULL
OR site = $1
)
AND (
@ -12,6 +11,7 @@ WHERE
OR NOT is_read
)
AND (
$3 :: text IS NULL
OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $3)
$3::text IS NULL
OR TO_TSVECTOR('english', search_summary)
@@ WEBSEARCH_TO_TSQUERY('english', $3)
)

8
server/sql/need-search-summary.sql Normal file
View File

@ -0,0 +1,8 @@
SELECT
p.id,
link,
clean_summary
FROM
post AS p
INNER JOIN feed AS f ON p.site = f.slug -- necessary to weed out nzb posts
WHERE search_summary IS NULL;

2
server/sql/threads.sql
View File

@ -16,7 +16,7 @@ WHERE
)
AND (
$5 :: text IS NULL
OR to_tsvector('english', summary) @@ websearch_to_tsquery('english', $5)
OR to_tsvector('english', search_summary) @@ websearch_to_tsquery('english', $5)
)
ORDER BY
date DESC,

4
server/src/bin/letterbox-server.rs
View File

@ -7,6 +7,7 @@ use std::{error::Error, io::Cursor, str::FromStr};
use async_graphql::{extensions, http::GraphiQLSource, EmptySubscription, Schema};
use async_graphql_rocket::{GraphQLQuery, GraphQLRequest, GraphQLResponse};
use cacher::FilesystemCacher;
#[cfg(feature = "tantivy")]
use letterbox_server::tantivy::TantivyConnection;
use letterbox_server::{
@ -220,9 +221,10 @@ async fn main() -> Result<(), Box<dyn Error>> {
#[cfg(feature = "tantivy")]
let tantivy_conn = TantivyConnection::new(&config.newsreader_tantivy_db_path)?;
let cacher = FilesystemCacher::new(&config.slurp_cache_path)?;
let schema = Schema::build(QueryRoot, Mutation, EmptySubscription)
.data(Notmuch::default())
.data(config)
.data(cacher)
.data(pool.clone());
#[cfg(feature = "tantivy")]

11
server/src/graphql.rs
View File

@ -5,6 +5,7 @@ use async_graphql::{
Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
SimpleObject, Union,
};
use cacher::FilesystemCacher;
use log::info;
use notmuch::Notmuch;
use serde::{Deserialize, Serialize};
@ -14,7 +15,7 @@ use tracing::instrument;
#[cfg(feature = "tantivy")]
use crate::tantivy::TantivyConnection;
use crate::{config::Config, newsreader, nm, Query};
use crate::{newsreader, nm, Query};
/// # Number of seconds since the Epoch
pub type UnixTime = isize;
@ -478,8 +479,8 @@ impl QueryRoot {
#[instrument(skip_all, fields(thread_id=thread_id, request_id=request_id()))]
async fn thread<'ctx>(&self, ctx: &Context<'ctx>, thread_id: String) -> Result<Thread, Error> {
let nm = ctx.data_unchecked::<Notmuch>();
let cacher = ctx.data_unchecked::<FilesystemCacher>();
let pool = ctx.data_unchecked::<PgPool>();
let config = ctx.data_unchecked::<Config>();
let debug_content_tree = ctx
.look_ahead()
.field("messages")
@ -487,7 +488,7 @@ impl QueryRoot {
.field("contentTree")
.exists();
if newsreader::is_newsreader_thread(&thread_id) {
Ok(newsreader::thread(config, pool, thread_id).await?)
Ok(newsreader::thread(cacher, pool, thread_id).await?)
} else {
Ok(nm::thread(nm, pool, thread_id, debug_content_tree).await?)
}
@ -609,11 +610,13 @@ impl Mutation {
#[instrument(skip_all, fields(request_id=request_id()))]
async fn refresh<'ctx>(&self, ctx: &Context<'ctx>) -> Result<bool, Error> {
let nm = ctx.data_unchecked::<Notmuch>();
let cacher = ctx.data_unchecked::<FilesystemCacher>();
let pool = ctx.data_unchecked::<PgPool>();
info!("{}", String::from_utf8_lossy(&nm.new()?));
newsreader::refresh(pool, cacher).await?;
#[cfg(feature = "tantivy")]
{
let tantivy = ctx.data_unchecked::<TantivyConnection>();
let pool = ctx.data_unchecked::<PgPool>();
// TODO: parallelize
tantivy.refresh(pool).await?;
}

13
server/src/lib.rs
View File

@ -19,6 +19,7 @@ use lol_html::{
RewriteStrSettings,
};
use maplit::{hashmap, hashset};
use regex::Regex;
use scraper::{Html, Selector};
use sqlx::types::time::PrimitiveDateTime;
use thiserror::Error;
@ -105,6 +106,8 @@ impl Transformer for StripHtml {
..RewriteStrSettings::default()
},
)?;
let re = Regex::new(r"\s+").expect("failed to parse regex");
let text = re.replace_all(&text, " ").to_string();
Ok(text)
}
@ -250,13 +253,13 @@ impl Transformer for AddOutlink {
}
}
struct SlurpContents {
cacher: Arc<Mutex<FilesystemCacher>>,
struct SlurpContents<'c> {
cacher: &'c FilesystemCacher,
inline_css: bool,
site_selectors: HashMap<String, Vec<Selector>>,
}
impl SlurpContents {
impl<'c> SlurpContents<'c> {
fn get_selectors(&self, link: &Url) -> Option<&[Selector]> {
for (host, selector) in self.site_selectors.iter() {
if link.host_str().map(|h| h.contains(host)).unwrap_or(false) {
@ -268,7 +271,7 @@ impl SlurpContents {
}
#[async_trait]
impl Transformer for SlurpContents {
impl<'c> Transformer for SlurpContents<'c> {
fn should_run(&self, link: &Option<Url>, html: &str) -> bool {
let mut will_slurp = false;
if let Some(link) = link {
@ -294,7 +297,7 @@ impl Transformer for SlurpContents {
let Some(selectors) = self.get_selectors(&link) else {
return Ok(html.to_string());
};
let cacher = self.cacher.lock().await;
let cacher = self.cacher;
let body = if let Some(body) = cacher.get(link.as_str()) {
String::from_utf8_lossy(&body).to_string()
} else {

175
server/src/newsreader.rs
View File

@ -1,4 +1,4 @@
use std::sync::Arc;
use std::collections::HashMap;
use cacher::FilesystemCacher;
use log::info;
@ -6,17 +6,15 @@ use maplit::hashmap;
use scraper::Selector;
use shared::compute_color;
use sqlx::postgres::PgPool;
use tokio::sync::Mutex;
use tracing::instrument;
use url::Url;
use crate::{
clean_title, compute_offset_limit,
config::Config,
error::ServerError,
graphql::{Corpus, NewsPost, Tag, Thread, ThreadSummary},
thread_summary_from_row, AddOutlink, FrameImages, Query, SanitizeHtml, SlurpContents,
ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX, NEWSREADER_THREAD_PREFIX,
StripHtml, ThreadSummaryRecord, Transformer, NEWSREADER_TAG_PREFIX, NEWSREADER_THREAD_PREFIX,
};
pub fn is_newsreader_query(query: &Query) -> bool {
@ -173,7 +171,7 @@ pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, Server
#[instrument(name = "newsreader::thread", skip_all, fields(thread_id=%thread_id))]
pub async fn thread(
config: &Config,
cacher: &FilesystemCacher,
pool: &PgPool,
thread_id: String,
) -> Result<Thread, ServerError> {
@ -191,73 +189,11 @@ pub async fn thread(
// TODO: remove the various places that have this as an Option
let link = Some(Url::parse(&r.link)?);
let mut body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
let cacher = Arc::new(Mutex::new(FilesystemCacher::new(&config.slurp_cache_path)?));
let body_tranformers: Vec<Box<dyn Transformer>> = vec![
let body_transformers: Vec<Box<dyn Transformer>> = vec![
Box::new(SlurpContents {
cacher,
inline_css: true,
site_selectors: hashmap![
"atmeta.com".to_string() => vec![
Selector::parse("div.entry-content").unwrap(),
],
"blog.prusa3d.com".to_string() => vec![
Selector::parse("article.content .post-block").unwrap(),
],
"blog.cloudflare.com".to_string() => vec![
Selector::parse(".author-lists .author-name-tooltip").unwrap(),
Selector::parse(".post-full-content").unwrap()
],
"blog.zsa.io".to_string() => vec![
Selector::parse("section.blog-article").unwrap(),
],
"engineering.fb.com".to_string() => vec![
Selector::parse("article").unwrap(),
],
"grafana.com".to_string() => vec![
Selector::parse(".blog-content").unwrap(),
],
"hackaday.com".to_string() => vec![
Selector::parse("div.entry-featured-image").unwrap(),
Selector::parse("div.entry-content").unwrap()
],
"ingowald.blog".to_string() => vec![
Selector::parse("article").unwrap(),
],
"jvns.ca".to_string() => vec![
Selector::parse("article").unwrap(),
],
"mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
"natwelch.com".to_string() => vec![
Selector::parse("article div.prose").unwrap(),
],
"rustacean-station.org".to_string() => vec![
Selector::parse("article").unwrap(),
],
"slashdot.org".to_string() => vec![
Selector::parse("span.story-byline").unwrap(),
Selector::parse("div.p").unwrap(),
],
"theonion.com".to_string() => vec![
// Single image joke w/ title
Selector::parse("article > section > div > figure").unwrap(),
// Single cartoon
Selector::parse("article > div > div > figure").unwrap(),
// Image at top of article
Selector::parse("article > header > div > div > figure").unwrap(),
// Article body
Selector::parse("article .entry-content > *").unwrap(),
],
"trofi.github.io".to_string() => vec![
Selector::parse("#content").unwrap(),
],
"www.redox-os.org".to_string() => vec![
Selector::parse("div.content").unwrap(),
],
"www.smbc-comics.com".to_string() => vec![
Selector::parse("img#cc-comic").unwrap(),
Selector::parse("div#aftercomic img").unwrap(),
],
],
site_selectors: slurp_contents_selectors(),
}),
Box::new(FrameImages),
Box::new(AddOutlink),
@ -268,7 +204,7 @@ pub async fn thread(
base_url: &link,
}),
];
for t in body_tranformers.iter() {
for t in body_transformers.iter() {
if t.should_run(&link, &body) {
body = t.transform(&link, &body).await?;
}
@ -316,3 +252,102 @@ pub async fn set_read_status<'ctx>(
}
Ok(true)
}
#[instrument(name = "newsreader::refresh", skip_all)]
pub async fn refresh<'ctx>(pool: &PgPool, cacher: &FilesystemCacher) -> Result<bool, ServerError> {
let body_transformers: Vec<Box<dyn Transformer>> = vec![
Box::new(SlurpContents {
cacher,
inline_css: true,
site_selectors: slurp_contents_selectors(),
}),
Box::new(StripHtml),
];
let rows = sqlx::query_file!("sql/need-search-summary.sql",)
.fetch_all(pool)
.await?;
for r in rows {
let link = Url::parse(&r.link)?;
info!("adding {link} to search index");
let link = Some(link);
let mut body = r.clean_summary.unwrap_or("NO SUMMARY".to_string());
for t in body_transformers.iter() {
if t.should_run(&link, &body) {
body = t.transform(&link, &body).await?;
}
}
sqlx::query!(
"UPDATE post SET search_summary = $1 WHERE id = $2",
body,
r.id
)
.execute(pool)
.await?;
}
Ok(true)
}
fn slurp_contents_selectors() -> HashMap<String, Vec<Selector>> {
hashmap![
"atmeta.com".to_string() => vec![
Selector::parse("div.entry-content").unwrap(),
],
"blog.prusa3d.com".to_string() => vec![
Selector::parse("article.content .post-block").unwrap(),
],
"blog.cloudflare.com".to_string() => vec![
Selector::parse(".author-lists .author-name-tooltip").unwrap(),
Selector::parse(".post-full-content").unwrap()
],
"blog.zsa.io".to_string() => vec![
Selector::parse("section.blog-article").unwrap(),
],
"engineering.fb.com".to_string() => vec![
Selector::parse("article").unwrap(),
],
"grafana.com".to_string() => vec![
Selector::parse(".blog-content").unwrap(),
],
"hackaday.com".to_string() => vec![
Selector::parse("div.entry-featured-image").unwrap(),
Selector::parse("div.entry-content").unwrap()
],
"ingowald.blog".to_string() => vec![
Selector::parse("article").unwrap(),
],
"jvns.ca".to_string() => vec![
Selector::parse("article").unwrap(),
],
"mitchellh.com".to_string() => vec![Selector::parse("div.w-full").unwrap()],
"natwelch.com".to_string() => vec![
Selector::parse("article div.prose").unwrap(),
],
"rustacean-station.org".to_string() => vec![
Selector::parse("article").unwrap(),
],
"slashdot.org".to_string() => vec![
Selector::parse("span.story-byline").unwrap(),
Selector::parse("div.p").unwrap(),
],
"theonion.com".to_string() => vec![
// Single image joke w/ title
Selector::parse("article > section > div > figure").unwrap(),
// Single cartoon
Selector::parse("article > div > div > figure").unwrap(),
// Image at top of article
Selector::parse("article > header > div > div > figure").unwrap(),
// Article body
Selector::parse("article .entry-content > *").unwrap(),
],
"trofi.github.io".to_string() => vec![
Selector::parse("#content").unwrap(),
],
"www.redox-os.org".to_string() => vec![
Selector::parse("div.content").unwrap(),
],
"www.smbc-comics.com".to_string() => vec![
Selector::parse("img#cc-comic").unwrap(),
Selector::parse("div#aftercomic img").unwrap(),
],
]
}

1
server/src/nm.rs
View File

@ -696,7 +696,6 @@ fn walk_attachments_inner<T, F: Fn(&ParsedMail, &[usize]) -> Option<T> + Copy>(
fn extract_attachments(m: &ParsedMail, id: &str) -> Result<Vec<Attachment>, ServerError> {
let mut attachments = Vec::new();
for (idx, sp) in m.subparts.iter().enumerate() {
info!("sp: {:?}", sp.headers);
if let Some(attachment) = extract_attachment(sp, id, &[idx]) {
// Filter out inline attachements, they're flattened into the body of the message.
if attachment.disposition == DispositionType::Attachment {