wathiede/letterbox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

server: explicitly reload tantivy reader after commit

Browse Source
This commit is contained in:
Bill Thiede 2024-12-16 08:34:35 -08:00
parent 7a1dec03a3
commit 4f4e474e66
1 changed files with 4 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
server/src/tantivy.rs
View File

@ -9,7 +9,7 @@ use tantivy::{
schema::{Facet, IndexRecordOption, Value}, schema::{Facet, IndexRecordOption, Value},
DocAddress, Index, IndexReader, Searcher, TantivyDocument, TantivyError, Term, DocAddress, Index, IndexReader, Searcher, TantivyDocument, TantivyError, Term,
}; };
use tracing::instrument; use tracing::{info_span, instrument};
use crate::{ use crate::{
compute_offset_limit, compute_offset_limit,
@ -110,13 +110,13 @@ impl TantivyConnection {
.collect(); .collect();
self.reindex_uids(pool, &uids).await self.reindex_uids(pool, &uids).await
} }
#[instrument(skip(self, pool))]
async fn reindex_uids(&self, pool: &PgPool, uids: &[String]) -> Result<(), ServerError> { async fn reindex_uids(&self, pool: &PgPool, uids: &[String]) -> Result<(), ServerError> {
if uids.is_empty() { if uids.is_empty() {
return Ok(()); return Ok(());
} }
// TODO: add SlurpContents and convert HTML to text // TODO: add SlurpContents and convert HTML to text
let start_time = std::time::Instant::now();
let pool: &PgPool = pool; let pool: &PgPool = pool;
let mut index_writer = self.index.writer(50_000_000)?; let mut index_writer = self.index.writer(50_000_000)?;
@ -145,13 +145,6 @@ impl TantivyConnection {
} }
let total = rows.len(); let total = rows.len();
for (i, r) in rows.into_iter().enumerate() { for (i, r) in rows.into_iter().enumerate() {
if i % 10_000 == 0 {
info!(
"{i}/{total} processed, elapsed {:.2}s",
start_time.elapsed().as_secs_f32()
);
}
let id_term = Term::from_field_text(uid, &r.uid); let id_term = Term::from_field_text(uid, &r.uid);
index_writer.delete_term(id_term); index_writer.delete_term(id_term);
let slug = r.site; let slug = r.site;
@ -170,9 +163,8 @@ impl TantivyConnection {
))?; ))?;
} }
info!("took {:.2}s to reindex", start_time.elapsed().as_secs_f32()); info_span!("IndexWriter.commit").in_scope(|| index_writer.commit())?;
info_span!("IndexReader.reload").in_scope(|| self.reader.reload())?;
index_writer.commit()?;
Ok(()) Ok(())
} }
#[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=%query))] #[instrument(name = "tantivy::reindex_thread", skip_all, fields(query=%query))]