Use env vars for worker counts

datman/src/bin/datman.rs

@@ -37,10 +37,6 @@ pub enum DatmanCommand {
 
         /// Name of the destination to back up.
         destination_name: String,
-
-        /// Number of workers to use for backup.
-        #[clap(long)]
-        num_workers: Option<u8>,
     },
 
     Extract {
@@ -71,10 +67,6 @@ pub enum DatmanCommand {
         /// Skip applying metadata. Might be needed to extract without superuser privileges.
         #[clap(long)]
         skip_metadata: bool,
-
-        /// Number of workers to use
-        #[clap(long)]
-        num_workers: Option<u8>,
     },
 }
 
@@ -125,7 +117,6 @@ fn main() -> anyhow::Result<()> {
         DatmanCommand::BackupOne {
             source_name,
             destination_name,
-            num_workers,
         } => {
             let descriptor = load_descriptor(Path::new(".")).unwrap();
             let source = &descriptor.source[&source_name];
@@ -137,7 +128,7 @@ fn main() -> anyhow::Result<()> {
                 Path::new("."),
                 &source_name,
                 &destination_name,
-                num_workers.unwrap_or(2),
+                yama::utils::get_number_of_workers("YAMA_CHUNKERS"),
             )
             .unwrap();
         }
@@ -149,7 +140,6 @@ fn main() -> anyhow::Result<()> {
             pile_name,
             destination,
             skip_metadata,
-            num_workers,
         } => {
             if !accept_partial {
                 bail!("Specify --accept-partial until running without it is supported.");
@@ -172,7 +162,7 @@ fn main() -> anyhow::Result<()> {
                 !skip_metadata,
                 !skip_metadata,
                 !skip_metadata,
-                num_workers.unwrap_or(2),
+                yama::utils::get_number_of_workers("YAMA_EXTRACTORS"),
             )?;
         }
     }

yama/src/commands.rs

@@ -15,6 +15,7 @@ use crate::pile::integrity::RawPileIntegrityChecker;
 use crate::pile::local_sqlitebloblogs::SqliteBloblogPile;
 use crate::pile::{Pile, PileDescriptor, PileStorage, RawPile};
 use crate::tree::{integrate_node_in_place, merge_uid_or_gid_tables};
+use crate::utils::get_number_of_workers;
 
 pub fn init(dir: &Path) -> anyhow::Result<()> {
     let yama_toml = dir.join("yama.toml");
@@ -53,6 +54,9 @@ pub fn load_pile_descriptor(dir: &Path) -> anyhow::Result<PileDescriptor> {
 }
 
 pub fn open_pile(dir: &Path, desc: &PileDescriptor) -> anyhow::Result<Pile<Box<dyn RawPile>>> {
+    let num_compressors = get_number_of_workers("YAMA_COMPRESSORS");
+    let num_decompressors = get_number_of_workers("YAMA_DECOMPRESSORS");
+
     match desc.storage {
         PileStorage::RemoteOnly => {
             bail!("This is a remote-only pile. No local storage allowed.");
@@ -73,8 +77,8 @@ pub fn open_pile(dir: &Path, desc: &PileDescriptor) -> anyhow::Result<Pile<Box<d
                         CompressionSettings {
                             dictionary: Arc::new(dictionary),
                             level: comp_level as i32,
-                            num_compressors: 4, // TODO make this configurable!
-                            num_decompressors: 4,
+                            num_compressors: num_compressors as u32,
+                            num_decompressors: num_decompressors as u32,
                         },
                     )?;
 

yama/src/utils.rs

@@ -7,3 +7,21 @@ pub fn bytes_to_hexstring(chunkid: &[u8]) -> String {
     }
     s
 }
+
+pub fn get_number_of_workers(first_try_env_name: &str) -> u8 {
+    let from_env_var = std::env::var(first_try_env_name)
+        .ok()
+        .or_else(|| std::env::var("YAMA_WORKERS").ok());
+    if let Some(from_env_var) = from_env_var {
+        from_env_var
+            .parse()
+            .expect("Number of workers should be an integer from 1 to 255.")
+    } else {
+        let number = num_cpus::get();
+        if number > u8::MAX.into() {
+            u8::MAX
+        } else {
+            number as u8
+        }
+    }
+}