Add structure to support raking icons

This commit is contained in:
Olivier 'reivilibre' 2022-03-26 17:15:00 +00:00
parent 8714f0ec80
commit eedf2ed183
2 changed files with 23 additions and 4 deletions

View File

@ -33,6 +33,7 @@ pub enum RakeOutcome {
RakedPage(RakedPage),
RakedFeed(Vec<UrlRaked>),
RakedSitemap(Vec<UrlRaked>),
RakedIcon(()),
Redirect {
reason: RedirectReason,
new_url: Url,
@ -252,7 +253,6 @@ impl Raker {
let content_type = content_type
.to_str()
.context("Can't convert content-type to str")?;
eprintln!("CT {:?}", content_type);
content_type.split(";").next().unwrap().trim().to_owned()
} else {
return Ok(RakeOutcome::TemporaryFailure(TemporaryFailure {
@ -299,6 +299,17 @@ impl Raker {
}
}
if intent == RakeIntent::Icon {
match rake_icon(&content, &content_type) {
Ok(icon) => {
return Ok(RakeOutcome::RakedIcon(icon));
}
Err(error) => {
debug!("Failed to rake as icon: {:?}", error);
}
}
}
return Ok(RakeOutcome::PermanentFailure(PermanentFailure {
reason: PermanentFailureReason::UnknownContentType(content_type.to_owned()),
}));
@ -432,6 +443,10 @@ pub fn rake_sitemap(content: &[u8]) -> anyhow::Result<Vec<UrlRaked>> {
Ok(urls)
}
pub fn rake_icon(content: &[u8], content_type: &str) -> anyhow::Result<()> {
todo!()
}
pub fn robots_txt_url_for(url: &Url) -> anyhow::Result<Url> {
url.join("/robots.txt")
.context("Whilst resolving /robots.txt on URL")

View File

@ -296,7 +296,7 @@ impl TaskContext {
RakeOutcome::RakedPage(page) => {
self.submission
.pages
.send((url.clone(), page.page_entry))
.send((url.clone(), page.page_entry.clone()))
.await
.context("Page processor shut down; can't stream page!")?;
@ -307,7 +307,7 @@ impl TaskContext {
.context("Reference processor shut down; can't stream references!")?;
self.as_event_processor()
.process_page(url.clone(), today)
.process_page(url.clone(), page.page_entry, today)
.await?;
self.as_event_processor()
.process_refs(url.clone(), page.referrer_entry, today)
@ -349,6 +349,10 @@ impl TaskContext {
Ok(NextAction::Continue)
}
RakeOutcome::RakedIcon(()) => {
// Store icon to icon store
todo!();
}
RakeOutcome::Redirect { reason, new_url } => {
let refs = RakedReferrerEntry {
references: [RakedReference {
@ -432,7 +436,7 @@ impl EventProcessor<'_> {
pub async fn process_page(
&self,
url: Url,
page: &RakedPageEntry,
page: RakedPageEntry,
datestamp: u16,
) -> anyhow::Result<()> {
self.store