diff --git a/src/dfa.rs b/src/dfa.rs index 8b846f4..7f46d54 100644 --- a/src/dfa.rs +++ b/src/dfa.rs @@ -7,6 +7,7 @@ use serde_derive::{Deserialize, Serialize}; pub enum Rule<'a> { Allow(&'a str), Disallow(&'a str), + #[cfg(feature = "crawl-delay")] Delay(&'a str), } @@ -15,6 +16,7 @@ impl<'a> Rule<'a> { match self { Rule::Allow(inner) => inner, Rule::Disallow(inner) => inner, + #[cfg(feature = "crawl-delay")] Rule::Delay(inner) => inner, } } diff --git a/src/parse.rs b/src/parse.rs index b1f7cf6..94b86da 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -5,6 +5,7 @@ use futures_util::{ }; use serde_derive::{Deserialize, Serialize}; const UA_PREFIX: &str = "user-agent:"; +#[cfg(feature = "crawl-delay")] const DELAY_PREFIX: &str = "crawl-delay:"; const ALLOW_PREFIX: &str = "allow:"; const DISALLOW_PREFIX: &str = "disallow:"; @@ -13,6 +14,7 @@ const DISALLOW_PREFIX: &str = "disallow:"; enum ParsedRule { Allow(String), Disallow(String), + #[cfg(feature = "crawl-delay")] Delay(String), } @@ -21,6 +23,7 @@ impl<'a> Into> for &'a ParsedRule { match self { ParsedRule::Allow(path) => Rule::Allow(&path[..]), ParsedRule::Disallow(path) => Rule::Disallow(&path[..]), + #[cfg(feature = "crawl-delay")] ParsedRule::Delay(delay) => Rule::Delay(delay), } } @@ -170,12 +173,21 @@ fn parse_line(line: String) -> ParsedLine { // This tries to parse lines roughly in order of most frequent kind to // least frequent kind in order to minimize CPU cycles on average. - parse_disallow(line) + + #[cfg(feature = "crawl-delay")] + return parse_disallow(line) .map(|s| ParsedLine::Rule(ParsedRule::Disallow(s.into()))) .or_else(|| parse_user_agent(line).map(|s| ParsedLine::UserAgent(s.to_lowercase()))) .or_else(|| parse_allow(line).map(|s| ParsedLine::Rule(ParsedRule::Allow(s.into())))) .or_else(|| parse_delay(line).map(|s| ParsedLine::Rule(ParsedRule::Delay(s.into())))) - .unwrap_or(ParsedLine::Nothing) + .unwrap_or(ParsedLine::Nothing); + + #[cfg(not(feature = "crawl-delay"))] + return parse_disallow(line) + .map(|s| ParsedLine::Rule(ParsedRule::Disallow(s.into()))) + .or_else(|| parse_user_agent(line).map(|s| ParsedLine::UserAgent(s.to_lowercase()))) + .or_else(|| parse_allow(line).map(|s| ParsedLine::Rule(ParsedRule::Allow(s.into())))) + .unwrap_or(ParsedLine::Nothing); } fn strip_comments(line: &str) -> &str { @@ -199,6 +211,7 @@ fn parse_user_agent(line: &str) -> Option<&str> { } } +#[cfg(feature = "crawl-delay")] fn parse_delay(line: &str) -> Option<&str> { if line.len() < DELAY_PREFIX.len() { return None;