wathiede/letterbox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

Merge news and email search results

Browse Source
This commit is contained in:
Bill Thiede
2024-08-06 20:44:25 -07:00
parent a84c9f0eaf
commit e570202ba2
9 changed files with 288 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
server/src/graphql.rs
View File

@@ -1,9 +1,11 @@
use async_graphql::{
connection::Connection, Context, EmptySubscription, Enum, Error, FieldResult, Object, Schema,
connection::{self, Connection, Edge, OpaqueCursor},
Context, EmptySubscription, Enum, Error, FieldResult, InputObject, Object, Schema,
SimpleObject, Union,
};
use log::info;
use notmuch::Notmuch;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgPool;
use crate::{newsreader, nm};
@@ -200,6 +202,12 @@ pub struct Tag {
pub unread: usize,
}
#[derive(Serialize, Deserialize, Debug, InputObject)]
struct SearchCursor {
newsreader_offset: i32,
notmuch_offset: i32,
}
pub struct QueryRoot;
#[Object]
impl QueryRoot {
@@ -207,12 +215,9 @@ impl QueryRoot {
let nm = ctx.data_unchecked::<Notmuch>();
let pool = ctx.data_unchecked::<PgPool>();
// TODO: make this search both copra and merge results
if newsreader::is_newsreader_search(&query) {
Ok(newsreader::count(pool, &query).await?)
} else {
Ok(nm::count(nm, &query).await?)
}
let newsreader_query: newsreader::Query = query.parse()?;
Ok(newsreader::count(pool, &newsreader_query).await? + nm::count(nm, &query).await?)
}
async fn search<'ctx>(
@@ -223,17 +228,117 @@ impl QueryRoot {
first: Option<i32>,
last: Option<i32>,
query: String,
) -> Result<Connection<usize, ThreadSummary>, Error> {
info!("search({after:?} {before:?} {first:?} {last:?} {query:?})");
) -> Result<Connection<OpaqueCursor<SearchCursor>, ThreadSummary>, Error> {
// TODO: add keywords to limit search to one corpus, i.e. is:news or is:mail
info!("search({after:?} {before:?} {first:?} {last:?} {query:?})",);
let nm = ctx.data_unchecked::<Notmuch>();
let pool = ctx.data_unchecked::<PgPool>();
// TODO: make this search both copra and merge results
if newsreader::is_newsreader_search(&query) {
Ok(newsreader::search(pool, after, before, first, last, query).await?)
} else {
Ok(nm::search(nm, after, before, first, last, query).await?)
enum ThreadSummaryCursor {
Newsreader(i32, ThreadSummary),
Notmuch(i32, ThreadSummary),
}
Ok(connection::query(
after,
before,
first,
last,
|after: Option<OpaqueCursor<SearchCursor>>,
before: Option<OpaqueCursor<SearchCursor>>,
first: Option<usize>,
last: Option<usize>| async move {
info!(
"search({:?} {:?} {first:?} {last:?} {query:?})",
after.as_ref().map(|v| &v.0),
before.as_ref().map(|v| &v.0)
);
let newsreader_after = after.as_ref().map(|sc| sc.newsreader_offset);
let notmuch_after = after.as_ref().map(|sc| sc.newsreader_offset);
let newsreader_before = before.as_ref().map(|sc| sc.newsreader_offset);
let notmuch_before = before.as_ref().map(|sc| sc.notmuch_offset);
let newsreader_query: newsreader::Query = query.parse()?;
let newsreader_results = newsreader::search(
pool,
newsreader_after,
newsreader_before,
first.map(|v| v as i32),
last.map(|v| v as i32),
&newsreader_query,
)
.await?
.into_iter()
.map(|(cur, ts)| ThreadSummaryCursor::Newsreader(cur, ts));
let notmuch_results = nm::search(
nm,
notmuch_after,
notmuch_before,
first.map(|v| v as i32),
last.map(|v| v as i32),
query,
)
.await?
.into_iter()
.map(|(cur, ts)| ThreadSummaryCursor::Notmuch(cur, ts));
let mut results: Vec<_> = newsreader_results.chain(notmuch_results).collect();
// The leading '-' is to reverse sort
results.sort_by_key(|item| match item {
ThreadSummaryCursor::Newsreader(_, ts) => -ts.timestamp,
ThreadSummaryCursor::Notmuch(_, ts) => -ts.timestamp,
});
let mut has_next_page = before.is_some();
if let Some(first) = first {
if results.len() > first {
has_next_page = true;
results.truncate(first);
}
}
let mut has_previous_page = after.is_some();
if let Some(last) = last {
if results.len() > last {
has_previous_page = true;
// TODO: find better way to do this.
results.reverse();
results.truncate(last);
results.reverse();
}
}
let mut connection = Connection::new(has_previous_page, has_next_page);
let mut newsreader_offset = 0;
let mut notmuch_offset = 0;
connection.edges.extend(results.into_iter().map(|item| {
let thread_summary;
match item {
ThreadSummaryCursor::Newsreader(offset, ts) => {
thread_summary = ts;
newsreader_offset = offset;
}
ThreadSummaryCursor::Notmuch(offset, ts) => {
thread_summary = ts;
notmuch_offset = offset;
}
}
info!(
"item: {} {}",
thread_summary.subject, thread_summary.timestamp
);
let cur = OpaqueCursor(SearchCursor {
newsreader_offset,
notmuch_offset,
});
Edge::new(cur, thread_summary)
}));
Ok::<_, async_graphql::Error>(connection)
},
)
.await?)
}
async fn tags<'ctx>(&self, ctx: &Context<'ctx>) -> FieldResult<Vec<Tag>> {

36
server/src/lib.rs
View File

@@ -345,3 +345,39 @@ pub fn sanitize_html(
Ok(clean_html)
}
fn compute_offset_limit(
after: Option<i32>,
before: Option<i32>,
first: Option<i32>,
last: Option<i32>,
) -> (i32, i32) {
let default_page_size = 100;
match (after, before, first, last) {
// Reasonable defaults
(None, None, None, None) => (0, default_page_size),
(None, None, Some(first), None) => (0, first),
(Some(after), None, None, None) => (after, default_page_size),
(Some(after), None, Some(first), None) => (after, first),
(None, Some(before), None, None) => (0.max(before - default_page_size), default_page_size),
(None, Some(before), None, Some(last)) => (0.max(before - last), last),
(None, None, None, Some(_)) => {
panic!("specifying last and no before doesn't make sense")
}
(None, None, Some(_), Some(_)) => {
panic!("specifying first and last doesn't make sense")
}
(None, Some(_), Some(_), _) => {
panic!("specifying before and first doesn't make sense")
}
(Some(_), Some(_), _, _) => {
panic!("specifying after and before doesn't make sense")
}
(Some(_), None, None, Some(_)) => {
panic!("specifying after and last doesn't make sense")
}
(Some(_), None, Some(_), Some(_)) => {
panic!("specifying after, first and last doesn't make sense")
}
}
}

163
server/src/newsreader.rs
View File

@@ -4,14 +4,16 @@ use std::{
str::FromStr,
};
use async_graphql::connection::{self, Connection, Edge};
use log::info;
use sqlx::postgres::PgPool;
use url::Url;
const TAG_PREFIX: &'static str = "News/";
const THREAD_PREFIX: &'static str = "news:";
const NON_EXISTENT_SITE_NAME: &'static str = "NO-SUCH-SITE";
use crate::{
compute_offset_limit,
error::ServerError,
graphql::{Body, Email, Html, Message, Tag, Thread, ThreadSummary},
EscapeHtml, InlineStyle, SanitizeHtml, Transformer,
@@ -25,10 +27,8 @@ pub fn is_newsreader_thread(query: &str) -> bool {
query.starts_with(THREAD_PREFIX)
}
pub async fn count(pool: &PgPool, query: &str) -> Result<usize, ServerError> {
let query: Query = query.parse()?;
let site = query.site.expect("search has no site");
let row = sqlx::query_file!("sql/count.sql", site, query.unread_only)
pub async fn count(pool: &PgPool, query: &Query) -> Result<usize, ServerError> {
let row = sqlx::query_file!("sql/count.sql", query.site, query.unread_only)
.fetch_one(pool)
.await?;
Ok(row.count.unwrap_or(0).try_into().unwrap_or(0))
@@ -36,102 +36,57 @@ pub async fn count(pool: &PgPool, query: &str) -> Result<usize, ServerError> {
pub async fn search(
pool: &PgPool,
after: Option<String>,
before: Option<String>,
after: Option<i32>,
before: Option<i32>,
first: Option<i32>,
last: Option<i32>,
query: String,
) -> Result<Connection<usize, ThreadSummary>, async_graphql::Error> {
let query: Query = query.parse()?;
let site = query.site.expect("search has no site");
connection::query(
after,
before,
first,
last,
|after: Option<usize>, before: Option<usize>, first, last| async move {
let default_page_size = 100;
let (offset, limit) = match (after, before, first, last) {
// Reasonable defaults
(None, None, None, None) => (0, default_page_size),
(None, None, Some(first), None) => (0, first),
(Some(after), None, None, None) => (after, default_page_size),
(Some(after), None, Some(first), None) => (after, first),
(None, Some(before), None, None) => {
(before.saturating_sub(default_page_size), default_page_size)
}
(None, Some(before), None, Some(last)) => (before.saturating_sub(last), last),
(None, None, None, Some(_)) => {
panic!("specifying last and no before doesn't make sense")
}
(None, None, Some(_), Some(_)) => {
panic!("specifying first and last doesn't make sense")
}
(None, Some(_), Some(_), _) => {
panic!("specifying before and first doesn't make sense")
}
(Some(_), Some(_), _, _) => {
panic!("specifying after and before doesn't make sense")
}
(Some(_), None, None, Some(_)) => {
panic!("specifying after and last doesn't make sense")
}
(Some(_), None, Some(_), Some(_)) => {
panic!("specifying after, first and last doesn't make sense")
}
};
// The +1 is to see if there are more pages of data available.
let limit = limit + 1;
query: &Query,
) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
info!("search({after:?} {before:?} {first:?} {last:?} {query:?}");
let (offset, limit) = compute_offset_limit(after, before, first, last);
// The +1 is to see if there are more pages of data available.
let limit = limit + 1;
info!("search offset {offset} limit {limit}");
let rows = sqlx::query_file!(
"sql/threads.sql",
site,
query.unread_only,
offset as i64,
limit as i64
)
.fetch_all(pool)
.await?;
let mut slice = rows
.into_iter()
.map(|r| {
let tags = if r.is_read.unwrap_or(false) {
vec![site.clone()]
} else {
vec!["unread".to_string(), site.clone()]
};
ThreadSummary {
thread: format!("{THREAD_PREFIX}{}", r.uid),
timestamp: r
.date
.expect("post missing date")
.assume_utc()
.unix_timestamp() as isize,
date_relative: "TODO date_relative".to_string(),
matched: 0,
total: 1,
authors: r.name.unwrap_or_else(|| site.clone()),
subject: r.title.unwrap_or("NO TITLE".to_string()),
tags,
}
})
.collect::<Vec<_>>();
let has_more = slice.len() == limit;
let mut connection = Connection::new(offset > 0, has_more);
if has_more {
slice.pop();
};
connection.edges.extend(
slice
.into_iter()
.enumerate()
.map(|(idx, item)| Edge::new(offset + idx, item)),
);
Ok::<_, async_graphql::Error>(connection)
},
let rows = sqlx::query_file!(
"sql/threads.sql",
query.site,
query.unread_only,
offset as i64,
limit as i64
)
.await
.fetch_all(pool)
.await?;
Ok(rows
.into_iter()
.enumerate()
.map(|(i, r)| {
let site = r.site.unwrap_or("UNKOWN SITE".to_string());
let tags = if r.is_read.unwrap_or(false) {
vec![site.clone()]
} else {
vec!["unread".to_string(), site.clone()]
};
(
i as i32 + offset,
ThreadSummary {
thread: format!("{THREAD_PREFIX}{}", r.uid),
timestamp: r
.date
.expect("post missing date")
.assume_utc()
.unix_timestamp() as isize,
date_relative: "TODO date_relative".to_string(),
matched: 0,
total: 1,
authors: r.name.unwrap_or_else(|| site.clone()),
subject: r.title.unwrap_or("NO TITLE".to_string()),
tags,
},
)
})
.collect())
}
pub async fn tags(pool: &PgPool, _needs_unread: bool) -> Result<Vec<Tag>, ServerError> {
@@ -258,11 +213,11 @@ pub async fn thread(pool: &PgPool, thread_id: String) -> Result<Thread, ServerEr
}
#[derive(Debug)]
struct Query {
unread_only: bool,
site: Option<String>,
uid: Option<String>,
remainder: Vec<String>,
pub struct Query {
pub unread_only: bool,
pub site: Option<String>,
pub uid: Option<String>,
pub remainder: Vec<String>,
}
impl FromStr for Query {
@@ -278,6 +233,10 @@ impl FromStr for Query {
unread_only = true
} else if word.starts_with(&site_prefix) {
site = Some(word[site_prefix.len()..].to_string())
} else if word.starts_with("tag:") {
// Any tag that doesn't match site_prefix should explicitly set the site to something not in the
// database
site = Some(NON_EXISTENT_SITE_NAME.to_string());
} else if word.starts_with(THREAD_PREFIX) {
uid = Some(word[THREAD_PREFIX.len()..].to_string())
} else {

65
server/src/nm.rs
View File

@@ -5,13 +5,13 @@ use std::{
time::Instant,
};
use async_graphql::connection::{self, Connection, Edge};
use log::{error, info, warn};
use mailparse::{parse_mail, MailHeader, MailHeaderMap, ParsedMail};
use memmap::MmapOptions;
use notmuch::Notmuch;
use crate::{
compute_offset_limit,
error::ServerError,
graphql::{
Attachment, Body, DispositionType, Email, Header, Html, Message, PlainText, Tag, Thread,
@@ -44,41 +44,22 @@ pub async fn count(nm: &Notmuch, query: &str) -> Result<usize, ServerError> {
pub async fn search(
nm: &Notmuch,
after: Option<String>,
before: Option<String>,
after: Option<i32>,
before: Option<i32>,
first: Option<i32>,
last: Option<i32>,
query: String,
) -> Result<Connection<usize, ThreadSummary>, async_graphql::Error> {
connection::query(
after,
before,
first,
last,
|after, before, first, last| async move {
let total = nm.count(&query)?;
let (first, last) = if let (None, None) = (first, last) {
info!("neither first nor last set, defaulting first to 20");
(Some(20), None)
} else {
(first, last)
};
let mut start = after.map(|after| after + 1).unwrap_or(0);
let mut end = before.unwrap_or(total);
if let Some(first) = first {
end = (start + first).min(end);
}
if let Some(last) = last {
start = if last > end - start { end } else { end - last };
}
let count = end - start;
let slice: Vec<ThreadSummary> = nm
.search(&query, start, count)?
.0
.into_iter()
.map(|ts| ThreadSummary {
) -> Result<Vec<(i32, ThreadSummary)>, async_graphql::Error> {
let (offset, limit) = compute_offset_limit(after, before, first, last);
Ok(nm
.search(&query, offset as usize, limit as usize)?
.0
.into_iter()
.enumerate()
.map(|(i, ts)| {
(
offset + i as i32,
ThreadSummary {
thread: format!("thread:{}", ts.thread),
timestamp: ts.timestamp,
date_relative: ts.date_relative,
@@ -87,20 +68,10 @@ pub async fn search(
authors: ts.authors,
subject: ts.subject,
tags: ts.tags,
})
.collect();
let mut connection = Connection::new(start > 0, end < total);
connection.edges.extend(
slice
.into_iter()
.enumerate()
.map(|(idx, item)| Edge::new(start + idx, item)),
);
Ok::<_, async_graphql::Error>(connection)
},
)
.await
},
)
})
.collect())
}
pub fn tags(nm: &Notmuch, needs_unread: bool) -> Result<Vec<Tag>, ServerError> {