93 lines
2.4 KiB
TOML
93 lines
2.4 KiB
TOML
[package]
|
|
name = "quickpeep_raker"
|
|
authors = ["Olivier 'reivilibre'"]
|
|
license = "MIT OR Apache-2.0"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
|
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
|
|
[dependencies]
|
|
### Subcrates
|
|
quickpeep_moz_readability = { path = "../quickpeep_moz_readability" }
|
|
quickpeep_densedoc = { path = "../quickpeep_densedoc" }
|
|
quickpeep_seed_parser = { path = "../quickpeep_seed_parser" }
|
|
quickpeep_structs = { path = "../quickpeep_structs" }
|
|
quickpeep_utils = { path = "../quickpeep_utils" }
|
|
|
|
### CLI Helpers
|
|
clap = { version = "3.1.6", features = ["derive"] }
|
|
colour = "0.6.0"
|
|
|
|
### Document Parsing
|
|
kuchiki = "0.8.1"
|
|
html5ever = "0.25.1"
|
|
markup5ever = "0.10.1"
|
|
serde = { version = "1.0.136", features = ["derive"] }
|
|
serde_bare = "0.5.0"
|
|
serde_json = "1.0.79"
|
|
|
|
ron = "0.7.0"
|
|
bytesize = {version = "1.1.0", features = ["serde"]}
|
|
|
|
### Dates
|
|
chrono = "0.4.19"
|
|
|
|
### Storage
|
|
sqlx = { version = "0.6.3", features = ["postgres", "runtime-tokio-rustls", "offline", "json"] }
|
|
# For compression of emitted packs. 0.11.1+zstd.1.5.2
|
|
zstd = "0.11.1"
|
|
|
|
### Utils
|
|
lazy_static = "1.4.0"
|
|
bytes = "1.1.0"
|
|
itertools = "0.10.3"
|
|
ipnetwork = "0.18.0"
|
|
futures-core = "0.3.28"
|
|
futures-util = "0.3.21"
|
|
tokio = { version = "1.17.0", features = ["full"] }
|
|
anyhow = "1.0.55"
|
|
log = "0.4.14"
|
|
env_logger = "0.9.0"
|
|
ouroboros = "0.14.2"
|
|
rand = "0.8.5"
|
|
lru = "0.7.3"
|
|
diplomatic-bag = "0.2.0"
|
|
arc-interner = "0.7.0"
|
|
smartstring = "1.0.0"
|
|
signal-hook = "0.3.13"
|
|
nix = "0.23.1"
|
|
quickpeep_html_charset_detection = { version = "0.1.0", path = "../quickpeep_html_charset_detection" }
|
|
tikv-jemallocator = "0.5.0"
|
|
|
|
### Raking helpers
|
|
# HTTP Requests
|
|
reqwest = { version = "0.11.9", features = ["stream"] }
|
|
# Gemini Requests
|
|
# N.B. TODO gemfeeds are Atom feeds for Gemini. Should support those.
|
|
gemini-fetch = "0.2.1"
|
|
# Robots.txt
|
|
cylon = { package = "reivilibre_fork_cylon", version = "0.2.1", features = ["crawl-delay"] }
|
|
# RSS/Atom/JSON feeds
|
|
feed-rs = "1.0.0"
|
|
# Sitemaps
|
|
sitemap = "0.4.1"
|
|
# Public Suffix List handling
|
|
publicsuffix = "2.1.1"
|
|
|
|
### Filtering helpers
|
|
# AdBlock
|
|
adblock = "0.5.0"
|
|
# Language detection
|
|
lingua = "1.3.3"
|
|
|
|
### Image processing
|
|
image = "0.24.1"
|
|
webp = { version = "0.2.2", features = ["img"] }
|
|
|
|
### Metrics
|
|
metrics = "0.18.1"
|
|
metrics-exporter-prometheus = { version = "0.9.0", default-features = false, features = ["http-listener"] }
|
|
metrics-process-promstyle = "0.18.0"
|
|
bare-metrics-recorder = "0.1.0"
|