From e25e92b273c9b38f39507fa50085c0b9df26b2a5 Mon Sep 17 00:00:00 2001 From: Olivier Date: Fri, 8 Jul 2022 15:06:30 +0100 Subject: [PATCH] Introduce 'exclusions' parameter to scanner --- datman/README.md | 1 + datman/src/commands/backup.rs | 4 ++-- datman/src/commands/ibrowse.rs | 3 ++- datman/src/commands/ilabel.rs | 3 ++- datman/src/remote/backup_source_responder.rs | 3 ++- datman/src/tree.rs | 21 +++++++++++++++----- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/datman/README.md b/datman/README.md index 4a27a60..934b50e 100644 --- a/datman/README.md +++ b/datman/README.md @@ -8,5 +8,6 @@ Features: * (optional) Compression using Zstd and a specifiable dictionary * (optional) Encryption * Ability to back up to remote machines over SSH +* Labelling of files in a backup source; different destinations can choose to backup either all or a subset of the labels. See the documentation for more information. diff --git a/datman/src/commands/backup.rs b/datman/src/commands/backup.rs index 0e77852..ecd8855 100644 --- a/datman/src/commands/backup.rs +++ b/datman/src/commands/backup.rs @@ -23,7 +23,7 @@ use anyhow::{anyhow, bail}; use arc_interner::ArcIntern; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use log::{info, warn}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; use std::io::Write; use std::path::Path; @@ -151,7 +151,7 @@ pub fn backup_source_to_destination( } => { info!("Looking to backup {} to {}", source_name, dest_name); info!("Scanning."); - let tree = scan(directory, !*cross_filesystems)? + let tree = scan(directory, !*cross_filesystems, &BTreeSet::new())? .ok_or_else(|| anyhow!("Source does not exist."))?; let absolute_source_path = desc_path.join(directory); diff --git a/datman/src/commands/ibrowse.rs b/datman/src/commands/ibrowse.rs index 37c0b92..3d35bb4 100644 --- a/datman/src/commands/ibrowse.rs +++ b/datman/src/commands/ibrowse.rs @@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public License along with Yama. If not, see . */ +use std::collections::BTreeSet; use std::path::Path; use anyhow::{anyhow, bail}; @@ -80,7 +81,7 @@ pub fn session(path: &Path, source_name: String) -> anyhow::Result<()> { }; println!("Scanning source; this might take a little while..."); - let mut dir_scan: FileTree1> = scan(directory, one_filesystem)? + let mut dir_scan: FileTree1> = scan(directory, one_filesystem, &BTreeSet::new())? .ok_or_else(|| anyhow!("Empty source."))? .replace_meta(&None); diff --git a/datman/src/commands/ilabel.rs b/datman/src/commands/ilabel.rs index 0d9ef41..21d507e 100644 --- a/datman/src/commands/ilabel.rs +++ b/datman/src/commands/ilabel.rs @@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public License along with Yama. If not, see . */ +use std::collections::BTreeSet; use std::io; use std::io::{StdinLock, Stdout, Write}; use std::path::Path; @@ -192,7 +193,7 @@ pub fn interactive_labelling_session(path: &Path, source_name: String) -> anyhow let my_hostname = get_hostname(); let mut dir_scan = if &my_hostname == hostname { info!("Scanning source; this might take a little while..."); - scan(directory, !*cross_filesystems)? + scan(directory, !*cross_filesystems, &BTreeSet::new())? .ok_or_else(|| anyhow!("Empty source."))? .replace_meta(&None) } else { diff --git a/datman/src/remote/backup_source_responder.rs b/datman/src/remote/backup_source_responder.rs index af28007..0966f62 100644 --- a/datman/src/remote/backup_source_responder.rs +++ b/datman/src/remote/backup_source_responder.rs @@ -1,6 +1,7 @@ // This file implements the responder side of the backup source protocol -- the protocol used // to connect to remote backup sources. +use std::collections::BTreeSet; use std::io::{stdin, stdout, Read, Write}; use std::path::PathBuf; use std::sync::Arc; @@ -43,7 +44,7 @@ pub fn introduction(read: &mut R, write: &mut W) -> anyhow::R pub fn scanning(read: &mut R, write: &mut W) -> anyhow::Result<()> { let path: PathBuf = read_message(read)?; let one_filesystem: bool = read_message(read)?; - let scan_result = scan(&path, one_filesystem)?; + let scan_result = scan(&path, one_filesystem, &BTreeSet::new())?; write_message(write, &scan_result)?; write.flush()?; Ok(()) diff --git a/datman/src/tree.rs b/datman/src/tree.rs index 0f6b46d..9409b38 100644 --- a/datman/src/tree.rs +++ b/datman/src/tree.rs @@ -15,12 +15,12 @@ You should have received a copy of the GNU General Public License along with Yama. If not, see . */ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Debug; use std::fs::{read_link, symlink_metadata, DirEntry, Metadata}; use std::io::ErrorKind; use std::os::unix::fs::MetadataExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::anyhow; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; @@ -216,14 +216,18 @@ pub fn mtime_msec(metadata: &Metadata) -> u64 { } /// Scan the filesystem to produce a Tree, using a default progress bar. -pub fn scan(path: &Path, one_filesystem: bool) -> anyhow::Result>> { +pub fn scan( + path: &Path, + one_filesystem: bool, + exclusions: &BTreeSet, +) -> anyhow::Result>> { let pbar = ProgressBar::with_draw_target(0, ProgressDrawTarget::stdout_with_hz(2)); pbar.set_style(ProgressStyle::default_spinner().template("{spinner} {pos:7} {msg}")); pbar.set_message("dir scan"); let one_filesystem = if one_filesystem { Some(None) } else { None }; - let result = scan_with_progress_bar(path, &pbar, one_filesystem); + let result = scan_with_progress_bar(path, &pbar, one_filesystem, exclusions); pbar.finish_at_current_pos(); result } @@ -233,7 +237,13 @@ pub fn scan_with_progress_bar( path: &Path, progress_bar: &ProgressBar, mut one_filesystem: Option>, + exclusions: &BTreeSet, ) -> anyhow::Result>> { + if exclusions.contains(path) { + // Don't enter excluded paths. + return Ok(None); + } + let metadata_res = symlink_metadata(path); progress_bar.inc(1); if let Err(e) = &metadata_res { @@ -305,7 +315,8 @@ pub fn scan_with_progress_bar( for entry in dir_read? { let entry: DirEntry = entry?; - let scanned = scan_with_progress_bar(&entry.path(), progress_bar, one_filesystem)?; + let scanned = + scan_with_progress_bar(&entry.path(), progress_bar, one_filesystem, exclusions)?; if let Some(scanned) = scanned { if let Ok(filename) = entry.file_name().into_string() { children.insert(filename, scanned);