From 626b4482454e77259c5eec86254e4835dba92c69 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 22 Mar 2023 23:08:08 +0000 Subject: [PATCH] raker: Switch to Jemalloc for the global allocator --- Cargo.lock | 21 +++++++++++++++++++++ quickpeep_raker/Cargo.toml | 1 + quickpeep_raker/src/bin/qp-raker.rs | 3 +++ 3 files changed, 25 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 945dafb..7c731ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3844,6 +3844,7 @@ dependencies = [ "sitemap", "smartstring", "tempfile", + "tikv-jemallocator", "tokio", "webp", "zstd", @@ -4879,6 +4880,26 @@ dependencies = [ "weezl", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.3+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a678df20055b43e57ef8cddde41cdfda9a3c1a060b67f4c5836dfb1d78543ba8" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20612db8a13a6c06d57ec83953694185a367e16945f66565e8028d2c0bd76979" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.1.44" diff --git a/quickpeep_raker/Cargo.toml b/quickpeep_raker/Cargo.toml index 868ecb0..0bee5fb 100644 --- a/quickpeep_raker/Cargo.toml +++ b/quickpeep_raker/Cargo.toml @@ -59,6 +59,7 @@ smartstring = "1.0.0" signal-hook = "0.3.13" nix = "0.23.1" quickpeep_html_charset_detection = { version = "0.1.0", path = "../quickpeep_html_charset_detection" } +tikv-jemallocator = "0.5.0" ### Raking helpers # HTTP Requests diff --git a/quickpeep_raker/src/bin/qp-raker.rs b/quickpeep_raker/src/bin/qp-raker.rs index f82f148..e202497 100644 --- a/quickpeep_raker/src/bin/qp-raker.rs +++ b/quickpeep_raker/src/bin/qp-raker.rs @@ -60,6 +60,9 @@ pub struct Opts { concurrent_sleepers: u32, } +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + #[tokio::main] pub async fn main() -> anyhow::Result<()> { env_logger::Builder::from_env(