Clean-up and allow worker number configuration

datman/src/bin/datman.rs

@@ -37,6 +37,10 @@ pub enum DatmanCommand {
 
         /// Name of the destination to back up.
         destination_name: String,
+
+        /// Number of workers to use for backup.
+        #[clap(long)]
+        num_workers: Option<u8>,
     },
 
     Extract {
@@ -67,6 +71,10 @@ pub enum DatmanCommand {
         /// Skip applying metadata. Might be needed to extract without superuser privileges.
         #[clap(long)]
         skip_metadata: bool,
+
+        /// Number of workers to use
+        #[clap(long)]
+        num_workers: Option<u8>,
     },
 }
 
@@ -117,6 +125,7 @@ fn main() -> anyhow::Result<()> {
         DatmanCommand::BackupOne {
             source_name,
             destination_name,
+            num_workers,
         } => {
             let descriptor = load_descriptor(Path::new(".")).unwrap();
             let source = &descriptor.source[&source_name];
@@ -128,6 +137,7 @@ fn main() -> anyhow::Result<()> {
                 Path::new("."),
                 &source_name,
                 &destination_name,
+                num_workers.unwrap_or(2),
             )
             .unwrap();
         }
@@ -139,6 +149,7 @@ fn main() -> anyhow::Result<()> {
             pile_name,
             destination,
             skip_metadata,
+            num_workers,
         } => {
             if !accept_partial {
                 bail!("Specify --accept-partial until running without it is supported.");
@@ -161,6 +172,7 @@ fn main() -> anyhow::Result<()> {
                 !skip_metadata,
                 !skip_metadata,
                 !skip_metadata,
+                num_workers.unwrap_or(2),
             )?;
         }
     }

datman/src/commands/backup.rs

@@ -53,6 +53,7 @@ pub fn backup_source_to_destination(
     desc_path: &Path,
     source_name: &str,
     dest_name: &str,
+    num_workers: u8,
 ) -> anyhow::Result<()> {
     match source {
         SourceDescriptor::DirectorySource {
@@ -149,13 +150,13 @@ pub fn backup_source_to_destination(
             }
 
             info!("Storing using yama.");
-            // TODO(configurability) num workers
             yama::operations::storing::store_fully(
                 &pile,
                 &absolute_source_path,
                 &pointer_name,
                 root,
                 parent,
+                num_workers,
             )?;
 
             info!("Stored!");

datman/src/commands/extract.rs

@@ -20,8 +20,8 @@ pub fn extract(
     apply_permissions: bool,
     apply_mtime: bool,
     apply_ownership: bool,
+    num_workers: u8,
 ) -> anyhow::Result<()> {
-    // TODO before and after support
     if destination.exists() {
         bail!("For now, the destination is not allowed to exist prior to extraction.");
     }
@@ -69,6 +69,7 @@ pub fn extract(
         apply_permissions,
         apply_mtime,
         apply_ownership,
+        num_workers,
     )?;
 
     Ok(())
@@ -141,10 +142,10 @@ fn extract_pointers_into_already_created_directory(
     apply_permissions: bool,
     apply_mtime: bool,
     apply_ownership: bool,
+    num_workers: u8,
 ) -> anyhow::Result<()> {
     for pointer in pointers {
         info!("Extracting {:?} now.", pointer);
-        // TODO(performance): configurable number of workers
 
         let pointer_target_dir = &target.join(&pointer);
         std::fs::create_dir(pointer_target_dir)?;
@@ -154,7 +155,7 @@ fn extract_pointers_into_already_created_directory(
             &pointer,
             pile,
             true,
-            2,
+            num_workers,
             apply_permissions,
             apply_mtime,
             apply_ownership,

yama/src/bin/yama.rs

@@ -40,6 +40,11 @@ enum PileCommand {
         subset: Vec<PathBuf>,
 
         destination: PathBuf,
+
+        /// Number of extraction workers to use. Ideal value varies, but probably not much more than
+        /// the number of CPU threads.
+        #[clap(long)]
+        num_workers: Option<u8>,
     },
     /// Check this yama pile for corruption.
     Check {
@@ -116,6 +121,7 @@ fn main() -> anyhow::Result<()> {
             pointer_name,
             subset,
             destination,
+            num_workers: workers,
         } => {
             let (_pdesc, pile) = open_pile()?;
             let mut pointer = pile
@@ -131,14 +137,13 @@ fn main() -> anyhow::Result<()> {
 
             fully_integrate_pointer_node(&pile, &mut root_tree_node.node, &mut pointer)?;
 
-            // todo >2 workers
             // todo allow disabling apply metadata
             extracting::extract(
                 destination,
                 &mut root_tree_node.node,
                 &pile,
                 true,
-                2,
+                workers.unwrap_or(2),
                 true,
                 true,
                 true,

yama/src/commands.rs

@@ -53,7 +53,6 @@ pub fn load_pile_descriptor(dir: &Path) -> anyhow::Result<PileDescriptor> {
     Ok(toml::from_slice(&buf)?)
 }
 
-// TODO with_name is weird and not a good name.
 pub fn open_pile(dir: &Path, desc: &PileDescriptor) -> anyhow::Result<Pile<Box<dyn RawPile>>> {
     match desc.storage {
         PileStorage::RemoteOnly => {

yama/src/operations/storing.rs

@@ -228,6 +228,7 @@ pub fn store_fully(
     new_pointer_name: &String,
     mut root_node: TreeNode,
     parent: Option<String>,
+    num_workers: u8,
 ) -> anyhow::Result<()> {
     if let Some(parent) = parent.as_ref() {
         let mut parent_pointer = pile.read_pointer(parent)?.ok_or_else(|| {
@@ -242,8 +243,7 @@ pub fn store_fully(
         differentiate_node_in_place(&mut root_node, &parent_node.node)?;
     }
 
-    // todo >2 workers
+    store(&root_dir, &mut root_node, &pile, true, num_workers)?;
-    store(&root_dir, &mut root_node, &pile, true, 2)?;
 
     let mut uid_lookup = BTreeMap::new();
     let mut gid_lookup = BTreeMap::new();

yama/src/tree.rs

@@ -181,68 +181,6 @@ pub fn integrate_node_in_place(new: &mut TreeNode, old: &TreeNode) -> anyhow::Re
     Ok(())
 }
 
-/// Encodes a node as a difference from a specified 'old' or base node.
-// pub fn differentiate_node(new: TreeNode, old: &TreeNode) -> anyhow::Result<TreeNode> {
-//     match new.content {
-//         TreeNode::Directory {
-//             children,
-//             ownership,
-//             permissions,
-//         } => match &old.content {
-//             TreeNode::Directory {
-//                 children: old_children,
-//                 ..
-//             } => {
-//                 // reformat into map for convenience.
-//                 let mut children = {
-//                     let mut map = BTreeMap::new();
-//                     for child in children.into_iter() {
-//                         map.insert(child.name.clone(), child);
-//                     }
-//                     map
-//                 };
-//
-//                 for old_child in old_children.iter() {
-//                     match children.entry(old_child.name.clone()) {
-//                         Entry::Vacant(ve) => {
-//                             ve.insert(TreeNode::Deleted);
-//                         }
-//                         Entry::Occupied(occ) => {
-//                             if !occ.get().metadata_invalidates(old_child, false) {
-//                                 occ.remove();
-//                             }
-//                             // leave it as it is TODO check this
-//                         }
-//                     }
-//                 }
-//
-//                 // repack into list
-//                 let mut children_vec = Vec::new();
-//                 for (_, subnode) in children.into_iter() {
-//                     children_vec.push(subnode);
-//                 }
-//                 Ok(TreeNode {
-//                     name: new.name,
-//                     content: TreeNode::Directory {
-//                         children: children_vec,
-//                         ownership,
-//                         permissions,
-//                     },
-//                 })
-//             }
-//             _ => Ok(TreeNode {
-//                 name: new.name,
-//                 content: TreeNode::Directory {
-//                     children,
-//                     ownership,
-//                     permissions,
-//                 },
-//             }),
-//         },
-//         _ => Ok(new),
-//     }
-// }
-
 /// Given a node, recursively constructs a UID and GID lookup table based on THIS system's
 /// users and groups.
 pub fn create_uidgid_lookup_tables(