Initial commit

.gitignore

@@ -0,0 +1,2 @@
+
+.idea

Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+members = [
+    "quickpeep",
+    "quickpeep_moz_readability"
+]
+

quickpeep/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "quickpeep"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+tokio = { version = "1.17.0", features = ["full"] }
+anyhow = "1.0.55"
+log = "0.4.14"
+env_logger = "0.9.0"
+quickpeep_moz_readability = { path = "../quickpeep_moz_readability" }
+
+# TODO: why do we need these here?
+kuchiki = "0.8.1"
+html5ever = "0.25.1"
+
+
+# TODO: rkyv and memmap2 should be an efficient way to load index packs into processes.
+# rkyv = "0.7.35"
+# memmap2 = "0.5.3"
+
+### Raking helpers
+# HTTP Requests
+reqwest = { version = "0.11.9", features = [] }
+# Gemini Requests
+# N.B. TODO gemfeeds are Atom feeds for Gemini. Should support those.
+gemini-fetch = "0.2.1"
+# Robots.txt
+cylon = { version = "0.2.0", features = [] }
+# RSS/Atom/JSON feeds
+feed-rs = "1.0.0"
+# Sitemaps
+sitemap = "0.4.1"
+
+### Filtering helpers
+# AdBlock
+adblock = "0.5.0"

quickpeep/src/bin/qp-rake.rs

@@ -0,0 +1,22 @@
+use quickpeep::raking::rake;
+use reqwest::Url;
+use std::str::FromStr;
+
+#[tokio::main]
+pub async fn main() -> anyhow::Result<()> {
+    let client = reqwest::Client::new();
+    // TODO max timeout, max body size
+    rake(
+        &Url::from_str("http://nothings.org/gamedev/ssao/")?,
+        &client,
+    )
+    .await?;
+
+    rake(
+        &Url::from_str("https://github.com/kuchiki-rs/kuchiki")?,
+        &client,
+    )
+    .await?;
+
+    Ok(())
+}

quickpeep/src/lib.rs

@@ -0,0 +1 @@
+pub mod raking;

quickpeep/src/raking.rs

@@ -0,0 +1,59 @@
+use anyhow::{bail, Context};
+use reqwest::header::HeaderValue;
+use reqwest::{Client, Url};
+
+pub enum RakeOutcome {
+    RakedPage(RakedPage),
+    RakedFeed(RakedFeed),
+    RakedSitemap(RakedSitemap),
+    TemporaryFailure(TemporaryFailure),
+    PermanentFailure(PermanentFailure),
+}
+
+pub struct RakedPage {}
+
+pub struct RakedFeed {}
+
+pub struct RakedSitemap {}
+
+pub struct TemporaryFailure {}
+
+pub struct PermanentFailure {
+    pub reason: PermanentFailureReason,
+}
+
+pub enum PermanentFailureReason {
+    ResourceDenied(u32),
+    WrongLanguage(String),
+}
+
+pub async fn rake(url: &Url, client: &Client) -> anyhow::Result<()> {
+    let response = client.get(url.clone()).send().await?;
+
+    if !response.status().is_success() {
+        bail!("Not successful: {:?}", response.status().as_u16());
+    }
+
+    if let Some(content_type) = response.headers().get("content-type") {
+        let content_type = content_type
+            .to_str()
+            .context("Can't convert content-type to str")?;
+        eprintln!("CT {:?}", content_type);
+    }
+
+    let content = response.bytes().await?;
+    let content_str = std::str::from_utf8(&content)?;
+
+    let mut readability = quickpeep_moz_readability::Readability::new(content_str);
+    readability
+        .parse(url.as_str())
+        .context("failed to analyse readability")?;
+
+    eprintln!("{:#?}", readability.metadata);
+
+    if let Some(node) = readability.article_node {
+        eprintln!("{}", node.to_string());
+    }
+
+    Ok(())
+}

quickpeep_moz_readability/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "quickpeep_moz_readability"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.56"
+kuchiki = "0.8.1"
+html5ever = "0.25.1"
+log = "0.4.14"
+url = "2.2.2"
+regex = "1.5.4"
+lazy_static = "1.4.0"

quickpeep_moz_readability/src/regexes.rs

@@ -0,0 +1,136 @@
+use lazy_static::lazy_static;
+/// This module contains regular expressions frequently used by quickpeep_moz_readability
+/// All regexes that only test if a `&str` matches the regex are preceded by the
+/// word "is_match". All other regexes are publicly accessible.
+use regex::Regex;
+pub fn is_match_byline(match_str: &str) -> bool {
+    lazy_static! {
+        static ref BYLINE_REGEX: Regex =
+            Regex::new(r"(?i)byline|author|dateline|writtenby|p-author").unwrap();
+    }
+    BYLINE_REGEX.is_match(match_str)
+}
+
+pub fn is_match_positive(match_str: &str) -> bool {
+    lazy_static! {
+        static ref POSITIVE_REGEX: Regex = Regex::new(r"(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story").unwrap();
+    }
+    POSITIVE_REGEX.is_match(match_str)
+}
+
+pub fn is_match_negative(match_str: &str) -> bool {
+    lazy_static! {
+        static ref NEGATIVE_REGEX: Regex = Regex::new(r"(?i)hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget").unwrap();
+    }
+    NEGATIVE_REGEX.is_match(match_str)
+}
+
+pub fn is_match_videos(match_str: &str) -> bool {
+    lazy_static! {
+        static ref VIDEOS_REGEX: Regex = Regex::new(r"(?i)//(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)").unwrap();
+    }
+    VIDEOS_REGEX.is_match(match_str)
+}
+
+pub fn is_match_unlikely(match_str: &str) -> bool {
+    lazy_static! {
+        static ref UNLIKELY_REGEX: Regex = Regex::new(r"(?i)-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote").unwrap();
+    }
+    UNLIKELY_REGEX.is_match(match_str)
+}
+
+pub fn is_match_ok_maybe(match_str: &str) -> bool {
+    lazy_static! {
+        static ref OK_MAYBE_REGEX: Regex =
+            Regex::new(r"(?i)and|article|body|column|content|main|shadow").unwrap();
+    }
+    OK_MAYBE_REGEX.is_match(match_str)
+}
+
+pub fn is_match_node_content(match_str: &str) -> bool {
+    lazy_static! {
+        static ref NODE_CONTENT_REGEX: Regex = Regex::new(r"\.( |$)").unwrap();
+    }
+    NODE_CONTENT_REGEX.is_match(match_str)
+}
+
+pub fn is_match_share_elems(match_str: &str) -> bool {
+    lazy_static! {
+        static ref SHARE_ELEMS_REGEX: Regex =
+            Regex::new(r"(?i)(\b|_)(share|sharedaddy)(\b|_)").unwrap();
+    }
+    SHARE_ELEMS_REGEX.is_match(match_str)
+}
+
+pub fn is_match_has_content(match_str: &str) -> bool {
+    lazy_static! {
+        static ref HAS_CONTENT_REGEX: Regex = Regex::new(r"\S$").unwrap();
+    }
+    HAS_CONTENT_REGEX.is_match(match_str)
+}
+
+pub fn is_match_img_ext(match_str: &str) -> bool {
+    lazy_static! {
+        static ref IMG_EXT_REGEX: Regex = Regex::new(r"(?i)\.(jpg|jpeg|png|webp)").unwrap();
+    }
+    IMG_EXT_REGEX.is_match(match_str)
+}
+
+pub fn is_match_srcset(match_str: &str) -> bool {
+    lazy_static! {
+        static ref SRCSET_REGEX: Regex = Regex::new(r"\.(jpg|jpeg|png|webp)\s+\d").unwrap();
+    }
+    SRCSET_REGEX.is_match(match_str)
+}
+
+pub fn is_match_src_regex(match_str: &str) -> bool {
+    lazy_static! {
+        static ref SRC_REGEX: Regex = Regex::new(r"^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$").unwrap();
+    }
+    SRC_REGEX.is_match(match_str)
+}
+
+pub fn is_match_name_pattern(match_str: &str) -> bool {
+    lazy_static! {
+        static ref NAME_PATTERN_REGEX: Regex = Regex::new(r"(?i)\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$").unwrap();
+    }
+    NAME_PATTERN_REGEX.is_match(match_str)
+}
+
+pub fn is_match_title_separator(match_str: &str) -> bool {
+    lazy_static! {
+        static ref TITLE_SEPARATOR_REGEX: Regex = Regex::new(r" [\|\-\\/>»] ").unwrap();
+    }
+    TITLE_SEPARATOR_REGEX.is_match(match_str)
+}
+
+pub fn is_match_has_title_separator(match_str: &str) -> bool {
+    lazy_static! {
+        static ref HAS_TITLE_SEPARATOR_REGEX: Regex = Regex::new(r" [\\/>»] ").unwrap();
+    }
+    HAS_TITLE_SEPARATOR_REGEX.is_match(match_str)
+}
+
+lazy_static! {
+    pub static ref NORMALIZE_REGEX: Regex = Regex::new(r"\s{2,}").unwrap();
+    pub static ref B64_DATA_URL_REGEX: Regex =
+        Regex::new(r"(?i)^data:\s*([^\s;,]+)\s*;\s*base64\s*").unwrap();
+    pub static ref BASE64_REGEX: Regex = Regex::new(r"(?i)base64\s*").unwrap();
+    pub static ref PROPERTY_REGEX: Regex = Regex::new(
+        r"(?i)\s*(dc|dcterm|og|twitter)\s*:\s*(author|creator|description|title|site_name)\s*"
+    )
+    .unwrap();
+    pub static ref SRCSET_CAPTURE_REGEX: Regex =
+        Regex::new(r"(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))").unwrap();
+    pub static ref REPLACE_WHITESPACE_REGEX: Regex = Regex::new(r"\s").unwrap();
+    pub static ref REPLACE_DOT_REGEX: Regex = Regex::new(r"\.").unwrap();
+    pub static ref REPLACE_HTML_ESCAPE_REGEX: Regex =
+        Regex::new("&(quot|amp|apos|lt|gt);").unwrap();
+    pub static ref REPLACE_HEX_REGEX: Regex =
+        Regex::new(r"(?i)&#(?:x([0-9a-z]{1,4})|([0-9]{1,4}));").unwrap();
+    pub static ref REPLACE_START_SEPARATOR_REGEX: Regex =
+        Regex::new(r"(?i)(?P<start>.*)[\|\-\\/>»] .*").unwrap();
+    pub static ref REPLACE_END_SEPARATOR_REGEX: Regex =
+        Regex::new(r"(?i)[^\|\-\\/>»]*[\|\-\\/>»](?P<end>.*)").unwrap();
+    pub static ref REPLACE_MULTI_SEPARATOR_REGEX: Regex = Regex::new(r"[\|\-\\/>»]+").unwrap();
+}

quickpeep_moz_readability/test_html/simple.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <title>Sample Document</title>
+</head>
+
+<body>
+    <h1>Some text in h1</h1>
+    <img src="inexistent.png">
+    <div class="invalid-elems">
+        <!-- This div contains invalid elements -->
+        <h1>Imagine some lorem ipsum</h1>
+        <img>
+    </div>
+    <!-- Test that the no-script content is copied over -->
+    <img src="lazy-load.png">
+    <noscript>
+        <div class="parent">
+            <img src="eager-load.png" id="lazy-load">
+        </div>
+    </noscript>
+</body>
+
+</html>