From 5ddb4452ffd81427c720730555d7e178274f1502 Mon Sep 17 00:00:00 2001 From: Bill Thiede Date: Thu, 19 Dec 2024 18:35:46 -0800 Subject: [PATCH] email2db: stub CLI --- Cargo.lock | 109 +++++++++++++------------------------ server/Cargo.toml | 9 +-- server/src/bin/email2db.rs | 95 +------------------------------- server/src/mail.rs | 35 +++++++++++- 4 files changed, 79 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89968a8..b731f12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,43 +187,43 @@ checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] name = "async-graphql" -version = "6.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298a5d587d6e6fdb271bf56af2dc325a80eb291fd0fc979146584b9a05494a8c" +checksum = "59fd6bd734afb8b6e4d0f84a3e77305ce0a7ccc60d70f6001cb5e1c3f38d8ff1" dependencies = [ "async-graphql-derive", "async-graphql-parser", "async-graphql-value", "async-stream", "async-trait", - "base64 0.13.1", + "base64 0.22.1", "bytes 1.9.0", "fast_chemail", "fnv", + "futures-timer", "futures-util", "handlebars", - "http 0.2.12", + "http 1.2.0", "indexmap 2.7.0", "log", "mime", - "multer 2.1.0", + "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", "serde_json", "serde_urlencoded", - "static_assertions", + "static_assertions_next", "tempfile", "thiserror 1.0.69", ] [[package]] name = "async-graphql-derive" -version = "6.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f329c7eb9b646a72f70c9c4b516c70867d356ec46cb00dcac8ad343fd006b0" +checksum = "ac38b4dd452d529d6c0248b51df23603f0a875770352e26ae8c346ce6c149b3e" dependencies = [ "Inflector", "async-graphql-parser", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "6.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6139181845757fd6a73fbb8839f3d036d7150b798db0e9bb3c6e83cdd65bd53b" +checksum = "42d271ddda2f55b13970928abbcbc3423cfc18187c60e8769b48f21a93b7adaa" dependencies = [ "async-graphql-value", "pest", @@ -250,9 +250,9 @@ dependencies = [ [[package]] name = "async-graphql-rocket" -version = "6.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10c5004043928e9ea8ca2faecc473e3c4fe4f5be259f63c9d735c9a0e4760c2b" +checksum = "7488bd0becd6b97c0ef7ef489c9f30621f9ea2c13a4cfa4e39b92b8567239945" dependencies = [ "async-graphql", "rocket", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "6.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323a5143f5bdd2030f45e3f2e0c821c9b1d36e79cf382129c64299c50a7f3750" +checksum = "aefe909173a037eaf3281b046dc22580b59a38b765d7b8d5116f2ffef098048d" dependencies = [ "bytes 1.9.0", "indexmap 2.7.0", @@ -1701,6 +1701,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -2022,9 +2028,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.5.0" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa67bab9ff362228eb3d00bd024a4965d8231bbb7921167f0cfa66c6626b225" +checksum = "d08485b96a0e6393e9e4d1b8d48cf74ad6c063cd905eb33f42c1ce3f0377539b" dependencies = [ "log", "pest", @@ -3248,24 +3254,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes 1.9.0", - "encoding_rs", - "futures-util", - "http 0.2.12", - "httparse", - "log", - "memchr", - "mime", - "spin", - "version_check", -] - [[package]] name = "multer" version = "3.1.0" @@ -4088,12 +4076,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.3.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "once_cell", - "toml_edit 0.19.15", + "toml_edit", ] [[package]] @@ -4550,7 +4537,7 @@ dependencies = [ "indexmap 2.7.0", "log", "memchr", - "multer 3.1.0", + "multer", "num_cpus", "parking_lot 0.12.3", "pin-project-lite", @@ -5173,6 +5160,7 @@ dependencies = [ "maplit", "memmap", "notmuch", + "opentelemetry", "reqwest 0.12.9", "rocket", "rocket_cors", @@ -5611,10 +5599,10 @@ dependencies = [ ] [[package]] -name = "static_assertions" -version = "1.1.0" +name = "static_assertions_next" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" [[package]] name = "string" @@ -5676,20 +5664,20 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.25.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -6407,7 +6395,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.22", + "toml_edit", ] [[package]] @@ -6419,17 +6407,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.7.0", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.22.22" @@ -6440,7 +6417,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.20", + "winnow", ] [[package]] @@ -7224,15 +7201,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.20" @@ -7283,7 +7251,6 @@ checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" [[package]] name = "xtracing" version = "0.1.0" -source = "git+http://git-private.h.xinu.tv/wathiede/xtracing.git#2802b8f6e2ef98895c703f1e6107a7396876f256" dependencies = [ "chrono", "if_empty", diff --git a/server/Cargo.toml b/server/Cargo.toml index 07a356d..b150d12 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -9,8 +9,8 @@ default-run = "server" [dependencies] ammonia = "3.3.0" anyhow = "1.0.79" -async-graphql = { version = "6.0.11", features = ["log"] } -async-graphql-rocket = "6.0.11" +async-graphql = { version = "7", features = ["log"] } +async-graphql-rocket = "7" async-trait = "0.1.81" build-info = "0.0.38" cacher = {git = "http://git-private.h.xinu.tv/wathiede/cacher.git"} @@ -25,6 +25,7 @@ mailparse = "0.15.0" maplit = "1.0.2" memmap = "0.7.0" notmuch = { path = "../notmuch" } +opentelemetry = "0.27.1" reqwest = { version = "0.12.7", features = ["blocking"] } rocket = { version = "0.5.0-rc.2", features = [ "json" ] } rocket_cors = "0.6.0" @@ -39,8 +40,8 @@ tokio = "1.26.0" tracing = "0.1.41" url = "2.5.2" urlencoding = "2.1.3" -#xtracing = { path = "../../xtracing" } -xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" } +xtracing = { path = "../../xtracing" } +#xtracing = { git = "http://git-private.h.xinu.tv/wathiede/xtracing.git" } [build-dependencies] build-info-build = "0.0.38" diff --git a/server/src/bin/email2db.rs b/server/src/bin/email2db.rs index 5fb11ec..27a4ac6 100644 --- a/server/src/bin/email2db.rs +++ b/server/src/bin/email2db.rs @@ -1,6 +1,5 @@ use chrono::NaiveDateTime; use clap::Parser; -use mailparse::{addrparse_header, dateparse, parse_mail, MailHeaderMap, ParsedMail}; use server::mail::read_mail_to_db; use sqlx::postgres::PgPool; @@ -16,99 +15,9 @@ struct Args { } #[tokio::main] async fn main() -> anyhow::Result<()> { + let _guard = xtracing::init(env!("CARGO_BIN_NAME"))?; let args = Args::parse(); let pool = PgPool::connect(&args.db_url).await?; - let mut buffer = Vec::new(); - read_mail_to_db(&pool, &args.path)?; + read_mail_to_db(&pool, &args.path).await?; Ok(()) } - -async fn add_post( - pool: &PgPool, - site: &str, - title: &str, - summary: &str, - link: &str, - date: i64, - uid: &str, - feed_url: &str, -) -> Result<(), MailError> { - todo!("add_post") - /* - // site | text - // title | text - // summary | text - // link | text - // date | timestamp without time zone - // is_read | boolean - // uid | text - // clean_site | text - // clean_title | text - // clean_summary | text - // clean_summary_text | text - // feed_url | text - let date = NaiveDateTime::from_timestamp_millis(date * 1000); - - sqlx::query!( - r#" - INSERT INTO post ( - site, - title, - summary, - link, - date, - is_read, - uid, - clean_title, - clean_summary, - clean_summary_text, - feed_url - ) - VALUES ( $1, $2, $3, $4, $5, false, $6, $2, $3, '', $7 ) - "#, - site, - title, - summary, - link, - date, - uid, - feed_url - ) - .execute(pool) - .await?; - Ok(()) - */ -} - -async fn find_feed(pool: &PgPool, name: &str, slug: &str, url: &str) -> Result { - match sqlx::query!( - r#" -SELECT id -FROM feed -WHERE slug = $1 - "#, - slug - ) - .fetch_one(pool) - .await - { - Err(sqlx::Error::RowNotFound) => { - let rec = sqlx::query!( - r#" -INSERT INTO feed ( name, slug, url, homepage, selector ) -VALUES ( $1, $2, $3, '', '' ) -RETURNING id - "#, - name, - slug, - url - ) - .fetch_one(pool) - .await?; - - return Ok(rec.id); - } - Ok(rec) => return Ok(rec.id), - Err(e) => return Err(e.into()), - }; -} diff --git a/server/src/mail.rs b/server/src/mail.rs index ff04cca..cc47189 100644 --- a/server/src/mail.rs +++ b/server/src/mail.rs @@ -5,6 +5,7 @@ use mailparse::{ }; use sqlx::postgres::PgPool; use thiserror::Error; +use tracing::info; #[derive(Error, Debug)] pub enum MailError { @@ -64,7 +65,7 @@ pub async fn read_mail_to_db(pool: &PgPool, path: &str) -> Result<(), MailError> if let Some(m) = first_html(&m) { let body = m.get_body()?; - add_post(&pool, &slug, &subject, &body, &message_id, date, &uid, &url).await?; + info!("add email {slug} {subject} {message_id} {date} {uid} {url}"); } else { return Err(MailError::MissingHtmlPart.into()); } @@ -79,3 +80,35 @@ fn first_html<'m>(m: &'m ParsedMail<'m>) -> Option<&'m ParsedMail<'m>> { } None } +async fn find_feed(pool: &PgPool, name: &str, slug: &str, url: &str) -> Result { + match sqlx::query!( + r#" +SELECT id +FROM feed +WHERE slug = $1 + "#, + slug + ) + .fetch_one(pool) + .await + { + Err(sqlx::Error::RowNotFound) => { + let rec = sqlx::query!( + r#" +INSERT INTO feed ( name, slug, url, homepage, selector ) +VALUES ( $1, $2, $3, '', '' ) +RETURNING id + "#, + name, + slug, + url + ) + .fetch_one(pool) + .await?; + + return Ok(rec.id); + } + Ok(rec) => return Ok(rec.id), + Err(e) => return Err(e.into()), + }; +}