diff --git a/datman/src/bin/datman.rs b/datman/src/bin/datman.rs index 8a55d61..d95ec4c 100644 --- a/datman/src/bin/datman.rs +++ b/datman/src/bin/datman.rs @@ -37,6 +37,10 @@ pub enum DatmanCommand { /// Name of the destination to back up. destination_name: String, + + /// Number of workers to use for backup. + #[clap(long)] + num_workers: Option, }, Extract { @@ -67,6 +71,10 @@ pub enum DatmanCommand { /// Skip applying metadata. Might be needed to extract without superuser privileges. #[clap(long)] skip_metadata: bool, + + /// Number of workers to use + #[clap(long)] + num_workers: Option, }, } @@ -117,6 +125,7 @@ fn main() -> anyhow::Result<()> { DatmanCommand::BackupOne { source_name, destination_name, + num_workers, } => { let descriptor = load_descriptor(Path::new(".")).unwrap(); let source = &descriptor.source[&source_name]; @@ -128,6 +137,7 @@ fn main() -> anyhow::Result<()> { Path::new("."), &source_name, &destination_name, + num_workers.unwrap_or(2), ) .unwrap(); } @@ -139,6 +149,7 @@ fn main() -> anyhow::Result<()> { pile_name, destination, skip_metadata, + num_workers, } => { if !accept_partial { bail!("Specify --accept-partial until running without it is supported."); @@ -161,6 +172,7 @@ fn main() -> anyhow::Result<()> { !skip_metadata, !skip_metadata, !skip_metadata, + num_workers.unwrap_or(2), )?; } } diff --git a/datman/src/commands/backup.rs b/datman/src/commands/backup.rs index bacecab..69f0787 100644 --- a/datman/src/commands/backup.rs +++ b/datman/src/commands/backup.rs @@ -53,6 +53,7 @@ pub fn backup_source_to_destination( desc_path: &Path, source_name: &str, dest_name: &str, + num_workers: u8, ) -> anyhow::Result<()> { match source { SourceDescriptor::DirectorySource { @@ -149,13 +150,13 @@ pub fn backup_source_to_destination( } info!("Storing using yama."); - // TODO(configurability) num workers yama::operations::storing::store_fully( &pile, &absolute_source_path, &pointer_name, root, parent, + num_workers, )?; info!("Stored!"); diff --git a/datman/src/commands/extract.rs b/datman/src/commands/extract.rs index 185203b..bc6e596 100644 --- a/datman/src/commands/extract.rs +++ b/datman/src/commands/extract.rs @@ -20,8 +20,8 @@ pub fn extract( apply_permissions: bool, apply_mtime: bool, apply_ownership: bool, + num_workers: u8, ) -> anyhow::Result<()> { - // TODO before and after support if destination.exists() { bail!("For now, the destination is not allowed to exist prior to extraction."); } @@ -69,6 +69,7 @@ pub fn extract( apply_permissions, apply_mtime, apply_ownership, + num_workers, )?; Ok(()) @@ -141,10 +142,10 @@ fn extract_pointers_into_already_created_directory( apply_permissions: bool, apply_mtime: bool, apply_ownership: bool, + num_workers: u8, ) -> anyhow::Result<()> { for pointer in pointers { info!("Extracting {:?} now.", pointer); - // TODO(performance): configurable number of workers let pointer_target_dir = &target.join(&pointer); std::fs::create_dir(pointer_target_dir)?; @@ -154,7 +155,7 @@ fn extract_pointers_into_already_created_directory( &pointer, pile, true, - 2, + num_workers, apply_permissions, apply_mtime, apply_ownership, diff --git a/yama/src/bin/yama.rs b/yama/src/bin/yama.rs index cd52f25..0cd51cf 100644 --- a/yama/src/bin/yama.rs +++ b/yama/src/bin/yama.rs @@ -40,6 +40,11 @@ enum PileCommand { subset: Vec, destination: PathBuf, + + /// Number of extraction workers to use. Ideal value varies, but probably not much more than + /// the number of CPU threads. + #[clap(long)] + num_workers: Option, }, /// Check this yama pile for corruption. Check { @@ -116,6 +121,7 @@ fn main() -> anyhow::Result<()> { pointer_name, subset, destination, + num_workers: workers, } => { let (_pdesc, pile) = open_pile()?; let mut pointer = pile @@ -131,14 +137,13 @@ fn main() -> anyhow::Result<()> { fully_integrate_pointer_node(&pile, &mut root_tree_node.node, &mut pointer)?; - // todo >2 workers // todo allow disabling apply metadata extracting::extract( destination, &mut root_tree_node.node, &pile, true, - 2, + workers.unwrap_or(2), true, true, true, diff --git a/yama/src/commands.rs b/yama/src/commands.rs index a38e3f1..8b5cca5 100644 --- a/yama/src/commands.rs +++ b/yama/src/commands.rs @@ -53,7 +53,6 @@ pub fn load_pile_descriptor(dir: &Path) -> anyhow::Result { Ok(toml::from_slice(&buf)?) } -// TODO with_name is weird and not a good name. pub fn open_pile(dir: &Path, desc: &PileDescriptor) -> anyhow::Result>> { match desc.storage { PileStorage::RemoteOnly => { diff --git a/yama/src/operations/storing.rs b/yama/src/operations/storing.rs index 58cc84c..54e9e8d 100644 --- a/yama/src/operations/storing.rs +++ b/yama/src/operations/storing.rs @@ -228,6 +228,7 @@ pub fn store_fully( new_pointer_name: &String, mut root_node: TreeNode, parent: Option, + num_workers: u8, ) -> anyhow::Result<()> { if let Some(parent) = parent.as_ref() { let mut parent_pointer = pile.read_pointer(parent)?.ok_or_else(|| { @@ -242,8 +243,7 @@ pub fn store_fully( differentiate_node_in_place(&mut root_node, &parent_node.node)?; } - // todo >2 workers - store(&root_dir, &mut root_node, &pile, true, 2)?; + store(&root_dir, &mut root_node, &pile, true, num_workers)?; let mut uid_lookup = BTreeMap::new(); let mut gid_lookup = BTreeMap::new(); diff --git a/yama/src/tree.rs b/yama/src/tree.rs index 8d2a60c..34522a7 100644 --- a/yama/src/tree.rs +++ b/yama/src/tree.rs @@ -181,68 +181,6 @@ pub fn integrate_node_in_place(new: &mut TreeNode, old: &TreeNode) -> anyhow::Re Ok(()) } -/// Encodes a node as a difference from a specified 'old' or base node. -// pub fn differentiate_node(new: TreeNode, old: &TreeNode) -> anyhow::Result { -// match new.content { -// TreeNode::Directory { -// children, -// ownership, -// permissions, -// } => match &old.content { -// TreeNode::Directory { -// children: old_children, -// .. -// } => { -// // reformat into map for convenience. -// let mut children = { -// let mut map = BTreeMap::new(); -// for child in children.into_iter() { -// map.insert(child.name.clone(), child); -// } -// map -// }; -// -// for old_child in old_children.iter() { -// match children.entry(old_child.name.clone()) { -// Entry::Vacant(ve) => { -// ve.insert(TreeNode::Deleted); -// } -// Entry::Occupied(occ) => { -// if !occ.get().metadata_invalidates(old_child, false) { -// occ.remove(); -// } -// // leave it as it is TODO check this -// } -// } -// } -// -// // repack into list -// let mut children_vec = Vec::new(); -// for (_, subnode) in children.into_iter() { -// children_vec.push(subnode); -// } -// Ok(TreeNode { -// name: new.name, -// content: TreeNode::Directory { -// children: children_vec, -// ownership, -// permissions, -// }, -// }) -// } -// _ => Ok(TreeNode { -// name: new.name, -// content: TreeNode::Directory { -// children, -// ownership, -// permissions, -// }, -// }), -// }, -// _ => Ok(new), -// } -// } - /// Given a node, recursively constructs a UID and GID lookup table based on THIS system's /// users and groups. pub fn create_uidgid_lookup_tables(