reivilibre
/
quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Simplify textification of indexed pages

This commit is contained in:
Olivier 'reivilibre' 2022-03-27 21:36:21 +01:00
parent 8ec8003dbb
commit eb899ac9a5
3 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
quickpeep_densedoc/src/lib.rs
View File

@ -66,67 +66,85 @@ impl DenseTree {
}
}
pub fn generate_textual_format(nodes: &Vec<DenseTree>) -> String {
pub fn generate_textual_format(nodes: &Vec<DenseTree>, rich: bool) -> String {
let mut buf = String::new();
for node in nodes {
node.append_in_textual_format(&mut buf);
node.append_in_textual_format(&mut buf, rich);
}
simplify_newlines(&buf)
}
fn append_in_textual_format(&self, string: &mut String) {
fn append_in_textual_format(&self, string: &mut String, rich: bool) {
match self {
DenseTree::Heading1(children) => {
string.push_str("\n\n# ");
if rich {
string.push_str("\n\n# ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Heading2(children) => {
string.push_str("\n\n## ");
if rich {
string.push_str("\n\n## ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Heading3(children) => {
string.push_str("\n\n### ");
if rich {
string.push_str("\n\n### ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Heading4(children) => {
string.push_str("\n\n#### ");
if rich {
string.push_str("\n\n#### ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Heading5(children) => {
string.push_str("\n\n##### ");
if rich {
string.push_str("\n\n##### ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Heading6(children) => {
string.push_str("\n\n###### ");
if rich {
string.push_str("\n\n###### ");
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str("\n");
}
DenseTree::Link { children, href, .. } => {
string.push('[');
if rich {
string.push('[');
};
for child in children {
child.append_in_textual_format(string);
child.append_in_textual_format(string, rich);
}
string.push_str(&format!("]({})", href));
if rich {
string.push_str(&format!("]({})", href));
};
}
DenseTree::Image { .. } => {
string.push_str("[IMG]");
if rich {
string.push_str("[IMG]");
};
}
DenseTree::Text(text) => {
string.push_str(text);

5
quickpeep_indexer/src/bin/qp-indexer.rs
View File

@ -76,8 +76,9 @@ pub async fn main() -> anyhow::Result<()> {
let document = page_record.record.document;
let article_body = DenseTree::generate_textual_format(&document.body_content);
let nonarticle_body = DenseTree::generate_textual_format(&document.body_remainder);
let article_body = DenseTree::generate_textual_format(&document.body_content, false);
let nonarticle_body =
DenseTree::generate_textual_format(&document.body_remainder, false);
let tags = seed_lookup
.look_up(&Url::parse(page_record.url.as_ref())?)?

4
quickpeep_raker/src/raking/page_extraction.rs
View File

@ -181,7 +181,9 @@ impl PageExtractionServiceInternal {
}
let dense_doc = DenseTree::from_body(root_node.clone());
let dense_doc_text = Lazy::new(Box::new(|| DenseTree::generate_textual_format(&dense_doc)));
let dense_doc_text = Lazy::new(Box::new(|| {
DenseTree::generate_textual_format(&dense_doc, true)
}));
//eprintln!("^^^^^\n{}\n^^^^^", *dense_doc_text);
if language.is_none() {