reivilibre
/
yama
1
0
Fork 0
Code Issues 27 Pull Requests Releases 16 Wiki Activity

Calculate chunk ID before submission

This commit is contained in:
Olivier 'reivilibre' 2021-11-20 11:02:07 +00:00
parent ff52dd74e7
commit 3c6c19e126
2 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
yama/src/chunking.rs
View File

@ -73,10 +73,10 @@ impl<'pile, RP: RawPile> RecursiveChunker<'pile, RP> {
let is_final = chunk.offset + chunk.length == self.buffer.len();
if !is_final || finalise {
consumed_until = Some(chunk.offset + chunk.length);
let chunk_id = self
.pile
.submit_chunk(&self.buffer[chunk.offset..chunk.offset + chunk.length])?;
let chunk_data = &self.buffer[chunk.offset..chunk.offset + chunk.length];
let chunk_id = calculate_chunkid(chunk_data);
new_chunks.extend_from_slice(&chunk_id);
self.pile.submit_chunk(chunk_id, chunk_data)?;
}
}
@ -101,7 +101,8 @@ impl<'pile, RP: RawPile> RecursiveChunker<'pile, RP> {
Ok(rcr)
} else {
// no chunking, so depth=0 (raw) and just emit our unchunked data
let chunk_id = self.pile.submit_chunk(&self.buffer)?;
let chunk_id = calculate_chunkid(&self.buffer);
self.pile.submit_chunk(chunk_id, &self.buffer)?;
Ok(RecursiveChunkRef { chunk_id, depth: 0 })
}
}

7
yama/src/pile.rs
View File

@ -19,7 +19,6 @@ use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use crate::chunking::calculate_chunkid;
use crate::definitions::{ChunkId, PointerData};
use crossbeam_channel::Sender;
use std::collections::HashSet;
@ -290,9 +289,7 @@ impl<R: RawPile> Pile<R> {
Ok(result)
}
pub fn submit_chunk(&self, chunk_data: &[u8]) -> anyhow::Result<ChunkId> {
let chunk_id = calculate_chunkid(chunk_data);
pub fn submit_chunk(&self, chunk_id: ChunkId, chunk_data: &[u8]) -> anyhow::Result<()> {
let mut racy_submissions = self.racy_submission_mutex.lock().unwrap();
if racy_submissions.insert(chunk_id) {
drop(racy_submissions);
@ -311,7 +308,7 @@ impl<R: RawPile> Pile<R> {
}
}
}
Ok(chunk_id)
Ok(())
}
/// Flushes buffered writes. Should really run this before exiting, so I can sleep better at