Cargo.lock

@@ -2519,7 +2519,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "letterbox"
-version = "0.0.61"
+version = "0.0.60"
 dependencies = [
  "build-info",
  "build-info-build",
@@ -3034,7 +3034,7 @@ dependencies = [
 
 [[package]]
 name = "notmuch"
-version = "0.0.61"
+version = "0.0.60"
 dependencies = [
  "itertools 0.10.5",
  "log",
@@ -3820,7 +3820,7 @@ dependencies = [
 
 [[package]]
 name = "procmail2notmuch"
-version = "0.0.61"
+version = "0.0.60"
 dependencies = [
  "anyhow",
 ]
@@ -4792,7 +4792,7 @@ dependencies = [
 
 [[package]]
 name = "server"
-version = "0.0.61"
+version = "0.0.60"
 dependencies = [
  "ammonia",
  "anyhow",
@@ -4892,7 +4892,7 @@ dependencies = [
 
 [[package]]
 name = "shared"
-version = "0.0.61"
+version = "0.0.60"
 dependencies = [
  "build-info",
  "notmuch",

notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "notmuch"
-version = "0.0.61"
+version = "0.0.60"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

notmuch/src/lib.rs

@@ -213,7 +213,7 @@ use std::{
     process::Command,
 };
 
-use log::{error, info};
+use log::info;
 use serde::{Deserialize, Serialize};
 
 /// # Number of seconds since the Epoch
@@ -507,19 +507,14 @@ impl Notmuch {
     ) -> Result<SearchSummary, NotmuchError> {
         let query = if query.is_empty() { "*" } else { query };
 
-        let res = self
-            .run_notmuch([
-                "search",
-                "--format=json",
-                &format!("--offset={offset}"),
-                &format!("--limit={limit}"),
-                query,
-            ])
-            .inspect_err(|err| error!("failed to notmuch search for query '{query}': {err}"))?;
-        Ok(serde_json::from_slice(&res).unwrap_or_else(|err| {
-            error!("failed to decode search result for query '{query}': {err}");
-            SearchSummary(Vec::new())
-        }))
+        let res = self.run_notmuch([
+            "search",
+            "--format=json",
+            &format!("--offset={offset}"),
+            &format!("--limit={limit}"),
+            query,
+        ])?;
+        Ok(serde_json::from_slice(&res)?)
     }
 
     pub fn count(&self, query: &str) -> Result<usize, NotmuchError> {
@@ -528,10 +523,8 @@ impl Notmuch {
         // let res = self.run_notmuch(["count", "--output=threads", query])?;
         let res = self.run_notmuch(["count", query])?;
         // Strip '\n' from res.
-        let s = std::str::from_utf8(&res)?.trim();
-        Ok(s.parse()
-            .inspect_err(|err| error!("failed to parse count for query '{query}': {err}"))
-            .unwrap_or(0))
+        let s = std::str::from_utf8(&res[..res.len() - 1])?;
+        Ok(s.parse()?)
     }
 
     pub fn show(&self, query: &str) -> Result<ThreadSet, NotmuchError> {

procmail2notmuch/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "procmail2notmuch"
-version = "0.0.61"
+version = "0.0.60"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "server"
-version = "0.0.61"
+version = "0.0.60"
 edition = "2021"
 default-run = "server"
 

server/src/graphql.rs

@@ -505,7 +505,7 @@ async fn tantivy_search(
 pub struct Mutation;
 #[Object]
 impl Mutation {
-    #[instrument(skip_all, fields(query=query, unread=unread))]
+    #[instrument(skip_all, fields(query, bool))]
     async fn set_read_status<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -522,7 +522,7 @@ impl Mutation {
         nm::set_read_status(nm, &query, unread).await?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag))]
+    #[instrument(skip_all, fields(query, tag))]
     async fn tag_add<'ctx>(
         &self,
         ctx: &Context<'ctx>,
@@ -534,7 +534,7 @@ impl Mutation {
         nm.tag_add(&tag, &query)?;
         Ok(true)
     }
-    #[instrument(skip_all, fields(query=query, tag=tag))]
+    #[instrument(skip_all, fields(query, tag))]
     async fn tag_remove<'ctx>(
         &self,
         ctx: &Context<'ctx>,

server/src/lib.rs

@@ -5,7 +5,7 @@ pub mod newsreader;
 pub mod nm;
 pub mod tantivy;
 
-use std::{collections::HashMap, convert::Infallible, fmt, str::FromStr, sync::Arc};
+use std::{collections::HashMap, convert::Infallible, str::FromStr, sync::Arc};
 
 use async_trait::async_trait;
 use cacher::{Cacher, FilesystemCacher};
@@ -612,38 +612,11 @@ pub struct Query {
     pub uids: Vec<String>,
     pub remainder: Vec<String>,
     pub is_notmuch: bool,
+    pub is_newsreader: bool,
     pub is_tantivy: bool,
     pub corpus: Option<Corpus>,
 }
 
-impl fmt::Display for Query {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        if self.unread_only {
-            write!(f, "is:unread ")?;
-        }
-        for tag in &self.tags {
-            write!(f, "tag:{tag} ")?;
-        }
-        for uid in &self.uids {
-            write!(f, "id:{uid} ")?;
-        }
-        if self.is_notmuch {
-            write!(f, "is:mail ")?;
-        }
-        if self.is_tantivy {
-            write!(f, "is:news ")?;
-        }
-        match self.corpus {
-            Some(c) => write!(f, "corpus:{c:?}")?,
-            _ => (),
-        }
-        for rem in &self.remainder {
-            write!(f, "{rem} ")?;
-        }
-        Ok(())
-    }
-}
-
 impl Query {
     // Converts the internal state of Query to something suitable for notmuch queries. Removes and
     // letterbox specific '<key>:<value' tags
@@ -675,6 +648,7 @@ impl FromStr for Query {
         let mut uids = Vec::new();
         let mut remainder = Vec::new();
         let mut is_notmuch = false;
+        let is_newsreader = false;
         let mut is_tantivy = false;
         let mut corpus = None;
         for word in s.split_whitespace() {
@@ -708,7 +682,10 @@ impl FromStr for Query {
             }
         }
         // If we don't see any explicit filters for a corpus, flip them all on
-        if corpus.is_none() && !(is_notmuch || is_tantivy) {
+        if corpus.is_none() && !(is_newsreader || is_notmuch || is_tantivy) {
+            // Don't set is_newsreader unless debugging, assume tantivy can handle it.
+            // Explicitely setting corpus:newsreader will by-pass this logic
+            // is_newsreader = true;
             is_notmuch = true;
             is_tantivy = true;
         }
@@ -718,6 +695,7 @@ impl FromStr for Query {
             uids,
             remainder,
             is_notmuch,
+            is_newsreader,
             is_tantivy,
             corpus,
         })

server/src/newsreader.rs

@@ -20,7 +20,7 @@ use crate::{
 };
 
 pub fn is_newsreader_query(query: &Query) -> bool {
-    query.corpus == Some(Corpus::Newsreader)
+    query.is_newsreader || query.corpus == Some(Corpus::Newsreader)
 }
 
 pub fn is_newsreader_thread(query: &str) -> bool {

server/src/nm.rs

@@ -49,7 +49,7 @@ pub fn threadset_to_messages(thread_set: notmuch::ThreadSet) -> Result<Vec<Messa
     Ok(Vec::new())
 }
 
-#[instrument(name="nm::count", skip_all, fields(query=%query))]
+#[instrument(name="nm::count", skip_all, fields(query=?query))]
 pub async fn count(nm: &Notmuch, query: &Query) -> Result<usize, ServerError> {
     if !is_notmuch_query(query) {
         return Ok(0);
@@ -58,7 +58,7 @@ pub async fn count(nm: &Notmuch, query: &Query) -> Result<usize, ServerError> {
     Ok(nm.count(&query)?)
 }
 
-#[instrument(name="nm::search", skip_all, fields(query=%query))]
+#[instrument(name="nm::search", skip_all, fields(query=?query))]
 pub async fn search(
     nm: &Notmuch,
     after: Option<i32>,
@@ -856,7 +856,7 @@ fn render_content_type_tree(m: &ParsedMail) -> String {
     )
 }
 
-#[instrument(name="nm::set_read_status", skip_all, fields(query=%query, unread=unread))]
+#[instrument(name="nm::set_read_status", skip_all, fields(query=?query, unread=unread))]
 pub async fn set_read_status<'ctx>(
     nm: &Notmuch,
     query: &Query,

server/src/tantivy.rs

@@ -7,7 +7,7 @@ use tantivy::{
     doc, query,
     query::{AllQuery, BooleanQuery, Occur, QueryParser, TermQuery},
     schema::{Facet, IndexRecordOption, Value},
-    DocAddress, Index, IndexReader, Searcher, TantivyDocument, TantivyError, Term,
+    DocAddress, Index, Searcher, TantivyDocument, TantivyError, Term,
 };
 use tracing::instrument;
 
@@ -24,29 +24,23 @@ pub fn is_tantivy_query(query: &Query) -> bool {
 }
 pub struct TantivyConnection {
     db_path: String,
-    index: Index,
-    reader: IndexReader,
-}
-
-fn get_index(db_path: &str) -> Result<Index, TantivyError> {
-    Ok(match Index::open_in_dir(db_path) {
-        Ok(idx) => idx,
-        Err(_) => {
-            create_news_db(db_path)?;
-            Index::open_in_dir(db_path)?
-        }
-    })
+    //index: Index,
 }
 
 impl TantivyConnection {
-    pub fn new(tantivy_db_path: &str) -> Result<TantivyConnection, TantivyError> {
-        let index = get_index(tantivy_db_path)?;
-        let reader = index.reader()?;
+    fn get_index(&self) -> Result<Index, TantivyError> {
+        Ok(match Index::open_in_dir(&self.db_path) {
+            Ok(idx) => idx,
+            Err(_) => {
+                create_news_db(&self.db_path)?;
+                Index::open_in_dir(&self.db_path)?
+            }
+        })
+    }
 
+    pub fn new(tantivy_db_path: &str) -> Result<TantivyConnection, TantivyError> {
         Ok(TantivyConnection {
             db_path: tantivy_db_path.to_string(),
-            index,
-            reader,
         })
     }
     #[instrument(name = "tantivy::refresh", skip_all)]
@@ -67,7 +61,7 @@ impl TantivyConnection {
         let start_time = std::time::Instant::now();
         let (searcher, _query) = self.searcher_and_query(&Query::default())?;
         let docs = searcher.search(&AllQuery, &DocSetCollector)?;
-        let uid = self.index.schema().get_field("uid")?;
+        let uid = self.get_index()?.schema().get_field("uid")?;
         let t_uids: Vec<_> = docs
             .into_iter()
             .map(|doc_address| {
@@ -118,8 +112,9 @@ impl TantivyConnection {
         let start_time = std::time::Instant::now();
         let pool: &PgPool = pool;
 
-        let mut index_writer = self.index.writer(50_000_000)?;
-        let schema = self.index.schema();
+        let index = self.get_index()?;
+        let mut index_writer = index.writer(50_000_000)?;
+        let schema = index.schema();
         let site = schema.get_field("site")?;
         let title = schema.get_field("title")?;
         let summary = schema.get_field("summary")?;
@@ -174,7 +169,7 @@ impl TantivyConnection {
         index_writer.commit()?;
         Ok(())
     }
-    #[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=%query))]
+    #[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=?query))]
     pub async fn reindex_thread(&self, pool: &PgPool, query: &Query) -> Result<(), ServerError> {
         let uids: Vec<_> = query
             .uids
@@ -198,19 +193,13 @@ impl TantivyConnection {
         &self,
         query: &Query,
     ) -> Result<(Searcher, Box<dyn query::Query>), ServerError> {
-        // TODO: only create one reader
-        // From https://tantivy-search.github.io/examples/basic_search.html
-        // "For a search server you will typically create one reader for the entire lifetime of
-        // your program, and acquire a new searcher for every single request."
-        //
-        // I think there's some challenge in making the reader work if we reindex, so reader my
-        // need to be stored indirectly, and be recreated on reindex
-        // I think creating a reader takes 200-300 ms.
-        let schema = self.index.schema();
-        let searcher = self.reader.searcher();
+        let index = self.get_index()?;
+        let reader = index.reader()?;
+        let schema = index.schema();
+        let searcher = reader.searcher();
         let title = schema.get_field("title")?;
         let summary = schema.get_field("summary")?;
-        let query_parser = QueryParser::for_index(&self.index, vec![title, summary]);
+        let query_parser = QueryParser::for_index(&index, vec![title, summary]);
         // Tantivy uses '*' to match all docs, not empty string
         let term = &query.remainder.join(" ");
         let term = if term.is_empty() { "*" } else { term };
@@ -218,8 +207,8 @@ impl TantivyConnection {
 
         let tantivy_query = query_parser.parse_query(&term)?;
 
-        let tag = schema.get_field("tag")?;
-        let is_read = schema.get_field("is_read")?;
+        let tag = self.get_index()?.schema().get_field("tag")?;
+        let is_read = self.get_index()?.schema().get_field("is_read")?;
         let mut terms = vec![(Occur::Must, tantivy_query)];
         for t in &query.tags {
             let facet = Facet::from(&format!("/{t}"));
@@ -239,7 +228,7 @@ impl TantivyConnection {
         Ok((searcher, Box::new(search_query)))
     }
 
-    #[instrument(name="tantivy::count", skip_all, fields(query=%query))]
+    #[instrument(name="tantivy::count", skip_all, fields(query=?query))]
     pub async fn count(&self, query: &Query) -> Result<usize, ServerError> {
         if !is_tantivy_query(query) {
             return Ok(0);
@@ -249,7 +238,7 @@ impl TantivyConnection {
         let (searcher, query) = self.searcher_and_query(&query)?;
         Ok(searcher.search(&query, &Count)?)
     }
-    #[instrument(name="tantivy::search", skip_all, fields(query=%query))]
+    #[instrument(name="tantivy::search", skip_all, fields(query=?query))]
     pub async fn search(
         &self,
         pool: &PgPool,
@@ -279,7 +268,7 @@ impl TantivyConnection {
                 .order_by_u64_field("date", tantivy::index::Order::Desc),
         )?;
         info!("search found {} docs", top_docs.len());
-        let uid = self.index.schema().get_field("uid")?;
+        let uid = self.get_index()?.schema().get_field("uid")?;
         let uids = top_docs
             .into_iter()
             .map(|(_, doc_address): (u64, DocAddress)| {

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "shared"
-version = "0.0.61"
+version = "0.0.60"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

web/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-version = "0.0.61"
+version = "0.0.60"
 name = "letterbox"
 repository = "https://github.com/seed-rs/seed-quickstart"
 authors = ["Bill Thiede <git@xinu.tv>"]