From 5d1f35a8ee16c9ff9218e1da2b2a7757f94a3e41 Mon Sep 17 00:00:00 2001
From: Olivier 'reivilibre <olivier@librepush.net>
Date: Sat, 11 Jun 2022 00:12:15 +0100
Subject: [PATCH] Deny content based on content-length header

---
 quickpeep_raker/src/raking.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/quickpeep_raker/src/raking.rs b/quickpeep_raker/src/raking.rs
index 32e7f13..a8ed0db 100644
--- a/quickpeep_raker/src/raking.rs
+++ b/quickpeep_raker/src/raking.rs
@@ -215,6 +215,26 @@ async fn response_to_bytes_limited(
     size_limit: usize,
     time_limit: Duration,
 ) -> anyhow::Result<Vec<u8>> {
+    // Check the content-length header without
+    let content_length = response
+        .headers()
+        .get("content-length")
+        .map(|len| len.to_str().ok())
+        .flatten()
+        .map(|len| len.parse::<u64>().ok())
+        .flatten();
+
+    if let Some(content_length) = content_length {
+        if content_length > size_limit as u64 {
+            // We can avoid downloading it: we already know it exceeds the limit.
+            increment_counter!("qprake_rake_specific_fail_count", "reason" => "SizeLimit");
+            return Err(PermanentFailure {
+                reason: PermanentFailureReason::ExceedsSizeLimit,
+            }
+            .into());
+        }
+    }
+
     let deadline = Instant::now() + time_limit;
     let mut buffer = Vec::new();
     let mut bytestream = response.bytes_stream();