reivilibre
/
yama
1
0
Fork 0
Code Issues 27 Pull Requests Releases 16 Wiki Activity

Calculate chunk ID before submission

This commit is contained in:
Olivier 'reivilibre' 2021-11-20 11:02:07 +00:00
parent ff52dd74e7
commit 3c6c19e126
2 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
yama/src/chunking.rs
View File

@ -73,10 +73,10 @@ impl<'pile, RP: RawPile> RecursiveChunker<'pile, RP> {
let is_final = chunk.offset + chunk.length == self.buffer.len(); let is_final = chunk.offset + chunk.length == self.buffer.len();
if !is_final || finalise { if !is_final || finalise {
consumed_until = Some(chunk.offset + chunk.length); consumed_until = Some(chunk.offset + chunk.length);
let chunk_id = self let chunk_data = &self.buffer[chunk.offset..chunk.offset + chunk.length];
.pile let chunk_id = calculate_chunkid(chunk_data);
.submit_chunk(&self.buffer[chunk.offset..chunk.offset + chunk.length])?;
new_chunks.extend_from_slice(&chunk_id); new_chunks.extend_from_slice(&chunk_id);
self.pile.submit_chunk(chunk_id, chunk_data)?;
} }
} }
@ -101,7 +101,8 @@ impl<'pile, RP: RawPile> RecursiveChunker<'pile, RP> {
Ok(rcr) Ok(rcr)
} else { } else {
// no chunking, so depth=0 (raw) and just emit our unchunked data // no chunking, so depth=0 (raw) and just emit our unchunked data
let chunk_id = self.pile.submit_chunk(&self.buffer)?; let chunk_id = calculate_chunkid(&self.buffer);
self.pile.submit_chunk(chunk_id, &self.buffer)?;
Ok(RecursiveChunkRef { chunk_id, depth: 0 }) Ok(RecursiveChunkRef { chunk_id, depth: 0 })
} }
} }

7
yama/src/pile.rs
View File

@ -19,7 +19,6 @@ use std::path::PathBuf;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::chunking::calculate_chunkid;
use crate::definitions::{ChunkId, PointerData}; use crate::definitions::{ChunkId, PointerData};
use crossbeam_channel::Sender; use crossbeam_channel::Sender;
use std::collections::HashSet; use std::collections::HashSet;
@ -290,9 +289,7 @@ impl<R: RawPile> Pile<R> {
Ok(result) Ok(result)
} }
pub fn submit_chunk(&self, chunk_data: &[u8]) -> anyhow::Result<ChunkId> { pub fn submit_chunk(&self, chunk_id: ChunkId, chunk_data: &[u8]) -> anyhow::Result<()> {
let chunk_id = calculate_chunkid(chunk_data);
let mut racy_submissions = self.racy_submission_mutex.lock().unwrap(); let mut racy_submissions = self.racy_submission_mutex.lock().unwrap();
if racy_submissions.insert(chunk_id) { if racy_submissions.insert(chunk_id) {
drop(racy_submissions); drop(racy_submissions);
@ -311,7 +308,7 @@ impl<R: RawPile> Pile<R> {
} }
} }
} }
Ok(chunk_id) Ok(())
} }
/// Flushes buffered writes. Should really run this before exiting, so I can sleep better at /// Flushes buffered writes. Should really run this before exiting, so I can sleep better at