move crawl-delay into a feature as it won't be user by many
This commit is contained in:
parent
fe11216642
commit
07a4ccadf0
@ -10,6 +10,9 @@ license = "MIT"
|
|||||||
keywords = ["robots", "txt", "parse", "compile"]
|
keywords = ["robots", "txt", "parse", "compile"]
|
||||||
repository = "https://github.com/crestonbunch/cylon"
|
repository = "https://github.com/crestonbunch/cylon"
|
||||||
|
|
||||||
|
[features]
|
||||||
|
crawl-delay = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
serde = "1.0"
|
serde = "1.0"
|
||||||
|
15
src/dfa.rs
15
src/dfa.rs
@ -1,3 +1,4 @@
|
|||||||
|
#[cfg(feature = "crawl-delay")]
|
||||||
use std::cmp::Ordering;
|
use std::cmp::Ordering;
|
||||||
|
|
||||||
use serde_derive::{Deserialize, Serialize};
|
use serde_derive::{Deserialize, Serialize};
|
||||||
@ -45,10 +46,12 @@ enum State {
|
|||||||
pub struct Cylon {
|
pub struct Cylon {
|
||||||
states: Vec<State>,
|
states: Vec<State>,
|
||||||
transitions: Vec<Vec<Transition>>,
|
transitions: Vec<Vec<Transition>>,
|
||||||
|
#[cfg(feature = "crawl-delay")]
|
||||||
delay: Option<u64>,
|
delay: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Cylon {
|
impl Cylon {
|
||||||
|
#[cfg(feature = "crawl-delay")]
|
||||||
pub fn delay(&self) -> Option<u64> {
|
pub fn delay(&self) -> Option<u64> {
|
||||||
self.delay
|
self.delay
|
||||||
}
|
}
|
||||||
@ -204,6 +207,8 @@ impl Cylon {
|
|||||||
transitions.push(t);
|
transitions.push(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "crawl-delay")]
|
||||||
|
{
|
||||||
let mut delays: Vec<Option<u64>> = rules.iter().filter(|rule| {
|
let mut delays: Vec<Option<u64>> = rules.iter().filter(|rule| {
|
||||||
match rule {
|
match rule {
|
||||||
Rule::Delay(_) => true,
|
Rule::Delay(_) => true,
|
||||||
@ -221,15 +226,19 @@ impl Cylon {
|
|||||||
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
delay: *delays.get(0).unwrap_or(&None),
|
delay: *delays.get(0).unwrap_or(&None),
|
||||||
states,
|
states,
|
||||||
transitions,
|
transitions,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "crawl-delay"))]
|
||||||
|
Self {
|
||||||
|
states,
|
||||||
|
transitions,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -312,6 +312,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[cfg(feature = "crawl-delay")]
|
||||||
fn test_crawl_delay() {
|
fn test_crawl_delay() {
|
||||||
tokio_test::block_on(async {
|
tokio_test::block_on(async {
|
||||||
let example_robots = r#"
|
let example_robots = r#"
|
||||||
|
Loading…
Reference in New Issue
Block a user