reivilibre
/
yama
1
0
Fork 0
Code Issues 27 Pull Requests Releases 16 Wiki Activity

Clean-up and allow worker number configuration

This commit is contained in:
Olivier 'reivilibre' 2021-06-16 12:43:07 +01:00
parent 2b0b090f9a
commit 09f65234a6
7 changed files with 27 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
datman/src/bin/datman.rs
View File

@ -37,6 +37,10 @@ pub enum DatmanCommand {
/// Name of the destination to back up.
destination_name: String,
/// Number of workers to use for backup.
#[clap(long)]
num_workers: Option<u8>,
},
Extract {
@ -67,6 +71,10 @@ pub enum DatmanCommand {
/// Skip applying metadata. Might be needed to extract without superuser privileges.
#[clap(long)]
skip_metadata: bool,
/// Number of workers to use
#[clap(long)]
num_workers: Option<u8>,
},
}
@ -117,6 +125,7 @@ fn main() -> anyhow::Result<()> {
DatmanCommand::BackupOne {
source_name,
destination_name,
num_workers,
} => {
let descriptor = load_descriptor(Path::new(".")).unwrap();
let source = &descriptor.source[&source_name];
@ -128,6 +137,7 @@ fn main() -> anyhow::Result<()> {
Path::new("."),
&source_name,
&destination_name,
num_workers.unwrap_or(2),
)
.unwrap();
}
@ -139,6 +149,7 @@ fn main() -> anyhow::Result<()> {
pile_name,
destination,
skip_metadata,
num_workers,
} => {
if !accept_partial {
bail!("Specify --accept-partial until running without it is supported.");
@ -161,6 +172,7 @@ fn main() -> anyhow::Result<()> {
!skip_metadata,
!skip_metadata,
!skip_metadata,
num_workers.unwrap_or(2),
)?;
}
}

3
datman/src/commands/backup.rs
View File

@ -53,6 +53,7 @@ pub fn backup_source_to_destination(
desc_path: &Path,
source_name: &str,
dest_name: &str,
num_workers: u8,
) -> anyhow::Result<()> {
match source {
SourceDescriptor::DirectorySource {
@ -149,13 +150,13 @@ pub fn backup_source_to_destination(
}
info!("Storing using yama.");
// TODO(configurability) num workers
yama::operations::storing::store_fully(
&pile,
&absolute_source_path,
&pointer_name,
root,
parent,
num_workers,
)?;
info!("Stored!");

7
datman/src/commands/extract.rs
View File

@ -20,8 +20,8 @@ pub fn extract(
apply_permissions: bool,
apply_mtime: bool,
apply_ownership: bool,
num_workers: u8,
) -> anyhow::Result<()> {
// TODO before and after support
if destination.exists() {
bail!("For now, the destination is not allowed to exist prior to extraction.");
}
@ -69,6 +69,7 @@ pub fn extract(
apply_permissions,
apply_mtime,
apply_ownership,
num_workers,
)?;
Ok(())
@ -141,10 +142,10 @@ fn extract_pointers_into_already_created_directory(
apply_permissions: bool,
apply_mtime: bool,
apply_ownership: bool,
num_workers: u8,
) -> anyhow::Result<()> {
for pointer in pointers {
info!("Extracting {:?} now.", pointer);
// TODO(performance): configurable number of workers
let pointer_target_dir = &target.join(&pointer);
std::fs::create_dir(pointer_target_dir)?;
@ -154,7 +155,7 @@ fn extract_pointers_into_already_created_directory(
&pointer,
pile,
true,
2,
num_workers,
apply_permissions,
apply_mtime,
apply_ownership,

9
yama/src/bin/yama.rs
View File

@ -40,6 +40,11 @@ enum PileCommand {
subset: Vec<PathBuf>,
destination: PathBuf,
/// Number of extraction workers to use. Ideal value varies, but probably not much more than
/// the number of CPU threads.
#[clap(long)]
num_workers: Option<u8>,
},
/// Check this yama pile for corruption.
Check {
@ -116,6 +121,7 @@ fn main() -> anyhow::Result<()> {
pointer_name,
subset,
destination,
num_workers: workers,
} => {
let (_pdesc, pile) = open_pile()?;
let mut pointer = pile
@ -131,14 +137,13 @@ fn main() -> anyhow::Result<()> {
fully_integrate_pointer_node(&pile, &mut root_tree_node.node, &mut pointer)?;
// todo >2 workers
// todo allow disabling apply metadata
extracting::extract(
destination,
&mut root_tree_node.node,
&pile,
true,
2,
workers.unwrap_or(2),
true,
true,
true,

1
yama/src/commands.rs
View File

@ -53,7 +53,6 @@ pub fn load_pile_descriptor(dir: &Path) -> anyhow::Result<PileDescriptor> {
Ok(toml::from_slice(&buf)?)
}
// TODO with_name is weird and not a good name.
pub fn open_pile(dir: &Path, desc: &PileDescriptor) -> anyhow::Result<Pile<Box<dyn RawPile>>> {
match desc.storage {
PileStorage::RemoteOnly => {

4
yama/src/operations/storing.rs
View File

@ -228,6 +228,7 @@ pub fn store_fully(
new_pointer_name: &String,
mut root_node: TreeNode,
parent: Option<String>,
num_workers: u8,
) -> anyhow::Result<()> {
if let Some(parent) = parent.as_ref() {
let mut parent_pointer = pile.read_pointer(parent)?.ok_or_else(|| {
@ -242,8 +243,7 @@ pub fn store_fully(
differentiate_node_in_place(&mut root_node, &parent_node.node)?;
}
// todo >2 workers
store(&root_dir, &mut root_node, &pile, true, 2)?;
store(&root_dir, &mut root_node, &pile, true, num_workers)?;
let mut uid_lookup = BTreeMap::new();
let mut gid_lookup = BTreeMap::new();

62
yama/src/tree.rs
View File

@ -181,68 +181,6 @@ pub fn integrate_node_in_place(new: &mut TreeNode, old: &TreeNode) -> anyhow::Re
Ok(())
}
/// Encodes a node as a difference from a specified 'old' or base node.
// pub fn differentiate_node(new: TreeNode, old: &TreeNode) -> anyhow::Result<TreeNode> {
// match new.content {
// TreeNode::Directory {
// children,
// ownership,
// permissions,
// } => match &old.content {
// TreeNode::Directory {
// children: old_children,
// ..
// } => {
// // reformat into map for convenience.
// let mut children = {
// let mut map = BTreeMap::new();
// for child in children.into_iter() {
// map.insert(child.name.clone(), child);
// }
// map
// };
//
// for old_child in old_children.iter() {
// match children.entry(old_child.name.clone()) {
// Entry::Vacant(ve) => {
// ve.insert(TreeNode::Deleted);
// }
// Entry::Occupied(occ) => {
// if !occ.get().metadata_invalidates(old_child, false) {
// occ.remove();
// }
// // leave it as it is TODO check this
// }
// }
// }
//
// // repack into list
// let mut children_vec = Vec::new();
// for (_, subnode) in children.into_iter() {
// children_vec.push(subnode);
// }
// Ok(TreeNode {
// name: new.name,
// content: TreeNode::Directory {
// children: children_vec,
// ownership,
// permissions,
// },
// })
// }
// _ => Ok(TreeNode {
// name: new.name,
// content: TreeNode::Directory {
// children,
// ownership,
// permissions,
// },
// }),
// },
// _ => Ok(new),
// }
// }
/// Given a node, recursively constructs a UID and GID lookup table based on THIS system's
/// users and groups.
pub fn create_uidgid_lookup_tables(