diff --git a/quickpeep/src/bin/qp-seedcoll-dump.rs b/quickpeep/src/bin/qp-seedcoll-dump.rs index 21e5942..4786628 100644 --- a/quickpeep/src/bin/qp-seedcoll-dump.rs +++ b/quickpeep/src/bin/qp-seedcoll-dump.rs @@ -1,14 +1,15 @@ use anyhow::{bail, Context}; use env_logger::Env; use futures_util::stream::StreamExt; +use itertools::Itertools; use quickpeep::config::WebConfig; use smartstring::alias::CompactString; use sqlx::sqlite::SqlitePoolOptions; use sqlx::{Connection, SqlitePool}; use std::collections::{BTreeSet, HashMap}; +use std::fs::OpenOptions; +use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; -use tokio::fs::OpenOptions; -use tokio::io::{AsyncWriteExt, BufWriter}; pub const DECISION_INCLUDED: i64 = 0; @@ -94,7 +95,7 @@ pub async fn seed_dump(pool: &SqlitePool, path: &Path) -> anyhow::Result<()> { .fetch_optional(&mut *txn) .await?; - let file = OpenOptions::new().create_new(true).open(path).await?; + let file = OpenOptions::new().create_new(true).write(true).open(path)?; let mut last_processed_position: Option = None; @@ -108,8 +109,8 @@ pub async fn seed_dump(pool: &SqlitePool, path: &Path) -> anyhow::Result<()> { WHERE collected_seed_id > ? AND decision = ? ORDER BY collected_seed_id ASC ", + process_from, DECISION_INCLUDED, - process_from ) .fetch(&mut *txn); @@ -122,7 +123,7 @@ pub async fn seed_dump(pool: &SqlitePool, path: &Path) -> anyhow::Result<()> { let mut tags: BTreeSet<&str> = row.tags.split(",").collect(); let diff_tags: Vec<&str> = row.tag_diff.split(",").collect(); - for diff_tag in diff_tags { + for diff_tag in diff_tags.into_iter().filter(|s| !s.is_empty()) { if diff_tag.starts_with('+') { tags.insert(&diff_tag[1..]); } else if diff_tag.starts_with('-') { @@ -146,17 +147,30 @@ pub async fn seed_dump(pool: &SqlitePool, path: &Path) -> anyhow::Result<()> { let mut buf_writer = BufWriter::new(file); - buf_writer.flush().await?; + for (tags, seeds) in seed_sets_to_seeds { + writeln!(buf_writer, "{}:", tags.iter().join(", "))?; + for seed in seeds { + if !seed.comment.is_empty() { + writeln!(buf_writer, "# {}", seed.comment.replace('\n', "\n# "))?; + } + writeln!(buf_writer, "{}", seed.url)?; + } + writeln!(buf_writer)?; + } - sqlx::query!( - " - REPLACE INTO seed_processing_positions (name, last_processed) - VALUES ('dumped', ?) - ", - last_processed_position - ) - .execute(&mut *txn) - .await?; + buf_writer.flush()?; + + if let Some(last_processed) = last_processed_position { + sqlx::query!( + " + REPLACE INTO seed_processing_positions (name, last_processed) + VALUES ('dumped', ?) + ", + last_processed + ) + .execute(&mut *txn) + .await?; + } txn.commit().await?; Ok(())