edgeparse 0.2.2__tar.gz → 0.2.4__tar.gz

{edgeparse-0.2.2 → edgeparse-0.2.4}/Cargo.lock

@@ -636,7 +636,7 @@ dependencies = [
 
 [[package]]
 name = "edgeparse-cli"
-version = "0.2.2"
+version = "0.2.4"
 dependencies = [
  "anyhow",
  "clap",
@@ -650,7 +650,7 @@ dependencies = [
 
 [[package]]
 name = "edgeparse-core"
-version = "0.2.2"
+version = "0.2.4"
 dependencies = [
  "anyhow",
  "base64",
@@ -674,7 +674,7 @@ dependencies = [
 
 [[package]]
 name = "edgeparse-node"
-version = "0.2.2"
+version = "0.2.4"
 dependencies = [
  "edgeparse-core",
  "napi",
@@ -686,7 +686,7 @@ dependencies = [
 
 [[package]]
 name = "edgeparse-python"
-version = "0.2.2"
+version = "0.2.4"
 dependencies = [
  "edgeparse-core",
  "pyo3",
@@ -695,7 +695,7 @@ dependencies = [
 
 [[package]]
 name = "edgeparse-wasm"
-version = "0.2.2"
+version = "0.2.4"
 dependencies = [
  "console_error_panic_hook",
  "console_log",

{edgeparse-0.2.2 → edgeparse-0.2.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/edgeparse-core", "crates/pdf-cos", "crates/edgeparse-python"]
 
 [workspace.package]
-version = "0.2.2"
+version = "0.2.4"
 edition = "2021"
 rust-version = "1.85"
 license = "Apache-2.0"

{edgeparse-0.2.2 → edgeparse-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: edgeparse
-Version: 0.2.2
+Version: 0.2.4
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License

{edgeparse-0.2.2 → edgeparse-0.2.4}/crates/edgeparse-core/src/api/config.rs

@@ -164,6 +164,21 @@ impl Default for ProcessingConfig {
     }
 }
 
+impl ProcessingConfig {
+    /// Returns true when a hybrid backend is configured.
+    pub fn hybrid_enabled(&self) -> bool {
+        !matches!(self.hybrid, HybridBackend::Off)
+    }
+
+    /// Returns true when local OCR recovery should run.
+    ///
+    /// OCR is only active in hybrid mode and can still be disabled explicitly
+    /// via `raster_table_ocr`.
+    pub fn raster_table_ocr_enabled(&self) -> bool {
+        self.raster_table_ocr && self.hybrid_enabled()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -182,4 +197,18 @@ mod tests {
         assert_eq!(config.hybrid, HybridBackend::Off);
         assert_eq!(config.hybrid_timeout, 30000);
     }
+
+    #[test]
+    fn test_raster_table_ocr_requires_hybrid_mode() {
+        let mut config = ProcessingConfig::default();
+        assert!(!config.hybrid_enabled());
+        assert!(!config.raster_table_ocr_enabled());
+
+        config.hybrid = HybridBackend::DoclingFast;
+        assert!(config.hybrid_enabled());
+        assert!(config.raster_table_ocr_enabled());
+
+        config.raster_table_ocr = false;
+        assert!(!config.raster_table_ocr_enabled());
+    }
 }

{edgeparse-0.2.2 → edgeparse-0.2.4}/crates/edgeparse-core/src/lib.rs

@@ -25,10 +25,12 @@ use crate::pdf::chunk_parser::extract_page_chunks;
 use crate::pdf::page_info;
 #[cfg(not(target_arch = "wasm32"))]
 use crate::pdf::raster_table_ocr::{
-    recover_page_raster_table_cell_text, recover_raster_table_borders,
+    recover_dominant_image_text_chunks, recover_page_raster_table_cell_text,
+    recover_raster_table_borders,
 };
 use crate::pipeline::orchestrator::{run_pipeline, PipelineState};
 use crate::tagged::struct_tree::build_mcid_map;
+use std::time::Instant;
 
 /// Main entry point: convert a PDF file to structured data.
 ///
@@ -46,23 +48,50 @@ pub fn convert(
     input_path: &std::path::Path,
     config: &ProcessingConfig,
 ) -> Result<PdfDocument, EdgePdfError> {
+    let timing_enabled = timing_enabled();
+    let total_start = Instant::now();
+
+    let phase_start = Instant::now();
     let raw_doc = pdf::loader::load_pdf(input_path, config.password.as_deref())?;
+    log_phase_duration(timing_enabled, "load_pdf", phase_start);
 
     // Extract per-page geometry (MediaBox, CropBox, rotation) for use throughout the pipeline.
+    let phase_start = Instant::now();
     let page_info_list = page_info::extract_page_info(&raw_doc.document);
+    log_phase_duration(timing_enabled, "extract_page_info", phase_start);
 
     // Extract text chunks from each page
     let pages_map = raw_doc.document.get_pages();
+    // Index by 1-based page number for fast lookup during optional OCR recovery.
+    // Keep this out of the default fast path when OCR is disabled.
+    let page_info_by_number: Vec<Option<&page_info::PageInfo>> =
+        if config.raster_table_ocr_enabled() {
+            let mut index = vec![None; pages_map.len().saturating_add(1)];
+            for info in &page_info_list {
+                if let Some(slot) = index.get_mut(info.page_number as usize) {
+                    *slot = Some(info);
+                }
+            }
+            index
+        } else {
+            Vec::new()
+        };
     let mut page_contents = Vec::with_capacity(pages_map.len());
 
+    let phase_start = Instant::now();
     for (&page_num, &page_id) in &pages_map {
         let page_chunks = extract_page_chunks(&raw_doc.document, page_num, page_id)?;
+        let mut recovered_text_chunks = Vec::new();
         let mut recovered_tables = Vec::new();
-        if config.raster_table_ocr {
-            if let Some(page_info) = page_info_list
-                .iter()
-                .find(|info| info.page_number == page_num)
-            {
+        if config.raster_table_ocr_enabled() {
+            if let Some(Some(page_info)) = page_info_by_number.get(page_num as usize) {
+                recovered_text_chunks = recover_dominant_image_text_chunks(
+                    input_path,
+                    &page_info.crop_box,
+                    page_num,
+                    &page_chunks.text_chunks,
+                    &page_chunks.image_chunks,
+                );
                 recovered_tables = recover_raster_table_borders(
                     input_path,
                     &page_info.crop_box,
@@ -77,6 +106,11 @@ pub fn convert(
             .into_iter()
             .map(ContentElement::TextChunk)
             .collect();
+        elements.extend(
+            recovered_text_chunks
+                .into_iter()
+                .map(ContentElement::TextChunk),
+        );
 
         elements.extend(
             page_chunks
@@ -104,12 +138,15 @@ pub fn convert(
 
         page_contents.push(elements);
     }
+    log_phase_duration(timing_enabled, "extract_page_chunks", phase_start);
 
     // Run the processing pipeline
+    let phase_start = Instant::now();
     let mcid_map = build_mcid_map(&raw_doc.document);
     let mut pipeline_state = PipelineState::with_mcid_map(page_contents, config.clone(), mcid_map)
-        .with_page_info(page_info_list.clone());
+        .with_page_info(page_info_list);
     run_pipeline(&mut pipeline_state)?;
+    log_phase_duration(timing_enabled, "run_pipeline", phase_start);
 
     // Build the output document
     let file_name = input_path
@@ -126,9 +163,10 @@ pub fn convert(
     doc.creation_date = raw_doc.metadata.creation_date;
     doc.modification_date = raw_doc.metadata.modification_date;
 
-    if config.raster_table_ocr {
+    let phase_start = Instant::now();
+    if config.raster_table_ocr_enabled() {
         for (page_idx, page) in pipeline_state.pages.iter_mut().enumerate() {
-            if let Some(page_info) = page_info_list.get(page_idx) {
+            if let Some(page_info) = pipeline_state.page_info.get(page_idx) {
                 recover_page_raster_table_cell_text(
                     input_path,
                     &page_info.crop_box,
@@ -138,11 +176,19 @@ pub fn convert(
             }
         }
     }
+    log_phase_duration(
+        timing_enabled,
+        "recover_page_raster_table_cell_text",
+        phase_start,
+    );
 
     // Flatten pipeline output into document kids
+    let phase_start = Instant::now();
     for page in pipeline_state.pages {
         doc.kids.extend(page);
     }
+    log_phase_duration(timing_enabled, "flatten_document", phase_start);
+    log_phase_duration(timing_enabled, "convert_total", total_start);
 
     Ok(doc)
 }
@@ -271,6 +317,27 @@ impl From<lopdf::Error> for EdgePdfError {
     }
 }
 
+fn timing_enabled() -> bool {
+    std::env::var("EDGEPARSE_TIMING")
+        .map(|value| {
+            matches!(
+                value.to_ascii_lowercase().as_str(),
+                "1" | "true" | "yes" | "on"
+            )
+        })
+        .unwrap_or(false)
+}
+
+fn log_phase_duration(enabled: bool, phase: &str, start: Instant) {
+    if enabled {
+        log::info!(
+            "Timing {}: {:.2} ms",
+            phase,
+            start.elapsed().as_secs_f64() * 1000.0
+        );
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

{edgeparse-0.2.2 → edgeparse-0.2.4}/crates/edgeparse-core/src/output/markdown.rs

@@ -208,6 +208,15 @@ fn render_markdown_core(doc: &PdfDocument) -> String {
                     continue;
                 }
 
+                // Demote carried-over table header rows that were promoted to
+                // headings by the pipeline but only duplicate the table above.
+                if looks_like_table_header_duplicate_heading(doc, i, trimmed) {
+                    output.push_str(&escape_md_line_start(trimmed));
+                    output.push_str("\n\n");
+                    i += 1;
+                    continue;
+                }
+
                 // Demote headings that sit in the bottom margin of the page
                 // (running footers misclassified as headings by the pipeline).
                 if looks_like_bottom_margin_heading(doc, i) {
@@ -1436,6 +1445,9 @@ fn render_layout_single_caption_chart_document_cached(
     if doc.number_of_pages != 1 {
         return None;
     }
+    if document_has_populated_table(doc) {
+        return None;
+    }
 
     let caption_indices = doc
         .kids
@@ -1546,6 +1558,22 @@ fn render_layout_single_caption_chart_document_cached(
     Some(output.trim_end().to_string() + "\n")
 }
 
+fn document_has_populated_table(doc: &PdfDocument) -> bool {
+    doc.kids.iter().any(|element| {
+        table_border_from_element(element).is_some_and(|table| {
+            table.num_rows >= 2
+                && table.num_columns >= 2
+                && table.rows.iter().any(|row| {
+                    row.cells
+                        .iter()
+                        .filter(|cell| !cell_text_content(cell).trim().is_empty())
+                        .count()
+                        >= 2
+                })
+        })
+    })
+}
+
 fn looks_like_chart_noise_element(_element: &ContentElement, text: &str) -> bool {
     if text.is_empty() {
         return false;
@@ -7814,6 +7842,9 @@ fn should_render_paragraph_as_heading(
     if looks_like_top_margin_running_header(doc, idx, text) {
         return false;
     }
+    if looks_like_hyphenated_table_title_continuation(doc, idx, text, next) {
+        return true;
+    }
     if should_render_element_as_heading(&doc.kids[idx], text, next) {
         return true;
     }
@@ -8379,6 +8410,135 @@ fn should_render_element_as_heading(
         && !is_attribution
 }
 
+fn looks_like_hyphenated_table_title_continuation(
+    doc: &PdfDocument,
+    idx: usize,
+    text: &str,
+    next: Option<&ContentElement>,
+) -> bool {
+    if !matches!(
+        next,
+        Some(ContentElement::Table(_)) | Some(ContentElement::TableBorder(_))
+    ) {
+        return false;
+    }
+
+    let trimmed = text.trim();
+    if trimmed.is_empty()
+        || starts_with_caption_prefix(trimmed)
+        || looks_like_numbered_section(trimmed)
+        || looks_like_keyword_numbered_section(trimmed)
+        || !trimmed.ends_with(':')
+    {
+        return false;
+    }
+
+    let word_count = trimmed.split_whitespace().count();
+    if !(3..=5).contains(&word_count) || trimmed.len() > 60 {
+        return false;
+    }
+
+    let Some(first_alpha) = trimmed.chars().find(|ch| ch.is_alphabetic()) else {
+        return false;
+    };
+    if first_alpha.is_lowercase() {
+        return false;
+    }
+
+    let Some(prev_idx) = idx.checked_sub(1) else {
+        return false;
+    };
+    let prev_text = extract_element_text(&doc.kids[prev_idx]);
+    let prev_trimmed = prev_text.trim();
+    !prev_trimmed.is_empty() && prev_trimmed.ends_with('-')
+}
+
+fn looks_like_table_header_duplicate_heading(doc: &PdfDocument, idx: usize, text: &str) -> bool {
+    let trimmed = text.trim();
+    if trimmed.is_empty()
+        || starts_with_caption_prefix(trimmed)
+        || looks_like_numbered_section(trimmed)
+        || looks_like_keyword_numbered_section(trimmed)
+    {
+        return false;
+    }
+
+    let word_count = trimmed.split_whitespace().count();
+    if !(3..=10).contains(&word_count) || trimmed.len() > 96 {
+        return false;
+    }
+
+    let Some(prev_idx) = idx.checked_sub(1) else {
+        return false;
+    };
+    let Some(previous_table) = table_border_from_element(&doc.kids[prev_idx]) else {
+        return false;
+    };
+    if previous_table.num_columns < 3 || previous_table.rows.len() < 3 {
+        return false;
+    }
+
+    let mut rendered_rows = collect_table_border_rows(previous_table);
+    if rendered_rows.is_empty() {
+        return false;
+    }
+    merge_continuation_rows(&mut rendered_rows);
+    trim_leading_table_carryover_rows(&mut rendered_rows);
+
+    let Some(header_row) = rendered_rows.first() else {
+        return false;
+    };
+    let header_text = header_row
+        .iter()
+        .map(|cell| cell.trim())
+        .filter(|cell| !cell.is_empty())
+        .collect::<Vec<_>>()
+        .join(" ");
+    if !equivalent_heading_text(trimmed, &header_text) {
+        return false;
+    }
+
+    let page_number = doc.kids[idx].page_number();
+    let mut short_fragments = 0usize;
+    let mut numeric_fragments = 0usize;
+
+    for candidate in doc.kids.iter().skip(idx + 1) {
+        if candidate.page_number() != page_number {
+            break;
+        }
+        if matches!(
+            candidate,
+            ContentElement::Table(_) | ContentElement::TableBorder(_)
+        ) {
+            break;
+        }
+
+        let fragment = extract_element_text(candidate);
+        let fragment_trimmed = fragment.trim();
+        if fragment_trimmed.is_empty()
+            || looks_like_margin_page_number(doc, candidate, fragment_trimmed)
+        {
+            continue;
+        }
+
+        let fragment_words = fragment_trimmed.split_whitespace().count();
+        if fragment_words > 6 {
+            return false;
+        }
+
+        short_fragments += 1;
+        if fragment_trimmed.chars().any(|ch| ch.is_ascii_digit()) {
+            numeric_fragments += 1;
+        }
+
+        if short_fragments >= 3 {
+            break;
+        }
+    }
+
+    short_fragments >= 2 && numeric_fragments >= 1
+}
+
 fn looks_like_top_margin_running_header(doc: &PdfDocument, idx: usize, text: &str) -> bool {
     let trimmed = text.trim();
     if trimmed.is_empty() || trimmed.split_whitespace().count() > 6 {
@@ -13033,6 +13193,32 @@ mod tests {
         assert!(md.contains("| kaolinite | 10 |"));
     }
 
+    #[test]
+    fn test_single_caption_chart_renderer_skips_documents_with_populated_tables() {
+        let mut doc = PdfDocument::new("table-with-caption.pdf".to_string());
+        doc.number_of_pages = 1;
+        for idx in 0..10 {
+            let bottom = 720.0 - idx as f64 * 18.0;
+            doc.kids.push(make_paragraph(
+                "Explanatory body text that should remain outside the chart-only renderer.",
+                bottom,
+                bottom + 10.0,
+            ));
+        }
+        doc.kids.push(make_paragraph(
+            "Figure 7.2: Kinematic Viscosity of Water at Atmospheric Pressure.",
+            150.0,
+            162.0,
+        ));
+        doc.kids.push(make_two_column_table(&[
+            ("Temperature", "Viscosity"),
+            ("20", "1.004"),
+            ("25", "0.893"),
+        ]));
+
+        assert!(render_layout_single_caption_chart_document(&doc).is_none());
+    }
+
     #[test]
     fn test_blank_right_column_table_is_not_misrendered_as_toc() {
         let mut doc = PdfDocument::new("flocculation-table.pdf".to_string());
@@ -13389,6 +13575,89 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn test_hyphenated_table_title_continuation_renders_as_heading() {
+        let mut doc = PdfDocument::new("hyphenated-table-title.pdf".to_string());
+        doc.number_of_pages = 1;
+        doc.kids.push(make_paragraph_at(
+            72.0,
+            724.0,
+            520.0,
+            738.0,
+            "With this in mind, here we have the 7 key competence areas selected to form a part of Eco-",
+        ));
+        doc.kids.push(make_paragraph_at(
+            72.0,
+            704.0,
+            260.0,
+            718.0,
+            "Circle's Competence Framework:",
+        ));
+        doc.kids.push(make_n_column_table(
+            &[
+                vec!["Eco-Circle Competence Framework"],
+                vec!["#1: The 3 Rs: Recycle-Reuse-Reduce"],
+                vec!["#2: Lifecycle of Circular Economy"],
+            ],
+            &[(140.0, 460.0)],
+        ));
+
+        let md = to_markdown(&doc).unwrap();
+        assert!(md.contains("# Circle's Competence Framework:"), "{md}");
+    }
+
+    #[test]
+    fn test_duplicate_table_header_heading_is_demoted() {
+        let mut doc = PdfDocument::new("duplicate-table-header-heading.pdf".to_string());
+        doc.number_of_pages = 1;
+        doc.kids
+            .push(make_heading("MOHAVE COMMUNITY COLLEGE BIO181"));
+        doc.kids.push(make_n_column_table(
+            &[
+                vec![
+                    "",
+                    "Saccharometer",
+                    "DI Water",
+                    "Glucose Solution",
+                    "Yeast Suspension",
+                ],
+                vec!["1", "", "8 ml", "6 ml", "0 ml"],
+                vec!["2", "", "12 ml", "0 ml", "2 ml"],
+                vec!["3", "", "6 ml", "6 ml", "2 ml"],
+            ],
+            &[
+                (72.0, 110.0),
+                (110.0, 210.0),
+                (210.0, 300.0),
+                (300.0, 430.0),
+                (430.0, 540.0),
+            ],
+        ));
+        doc.kids.push(make_heading_at(
+            72.0,
+            92.0,
+            390.0,
+            108.0,
+            "Saccharometer DI Water Glucose Solution Yeast Suspension",
+        ));
+        doc.kids
+            .push(make_paragraph_at(72.0, 72.0, 120.0, 88.0, "below"));
+        doc.kids
+            .push(make_paragraph_at(72.0, 56.0, 240.0, 72.0, "1 16 ml 12 ml"));
+        doc.kids
+            .push(make_paragraph_at(296.0, 56.0, 340.0, 72.0, "0 ml"));
+
+        let md = to_markdown(&doc).unwrap();
+        assert!(
+            md.contains("Saccharometer DI Water Glucose Solution Yeast Suspension"),
+            "{md}"
+        );
+        assert!(
+            !md.contains("# Saccharometer DI Water Glucose Solution Yeast Suspension"),
+            "{md}"
+        );
+    }
+
     #[test]
     fn test_geometric_panel_headers_are_promoted_into_table() {
         let mut doc = PdfDocument::new("ai-pack-panel.pdf".to_string());