graph-knowledge-doc-parser 0.2.2__tar.gz → 0.2.4__tar.gz

{graph_knowledge_doc_parser-0.2.2 → graph_knowledge_doc_parser-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graph-knowledge-doc-parser
-Version: 0.2.2
+Version: 0.2.4
 Summary: doc parser using llm driven engine with graph knowledge awareness
 Author: humblemat810
 Author-email: 67593116+humblemat810@users.noreply.github.com
@@ -65,6 +65,19 @@ The reusable helpers live under `src/workflow_ingest/` and are designed so the
 same core logic can be called from tests, scripts, and higher-level workflow
 code without duplicating orchestration.
 
+Layerwise parsing selects a strategy independently for each frontier layer.
+The default deterministic order is `layer_excerpt`, `layer_boundary`, then
+`page_index`; callers may provide another complete order, and optional triage
+may choose among the currently enabled strategies. Failed strategies are
+disabled for that layer and the workflow routes through the remaining methods
+before reaching explicit parse failure. PageIndex is a one-layer structural
+fallback that preserves exact source pointers and returns expandable children
+to normal strategy selection. See the
+[`0.2.4` release note](doc/release_0.2.4.md) and the
+[progressive refinement ADR](doc/adr_progressive_refinement_strategy_arbitration.md).
+The deterministic and text-only Bonsai fixture evaluation is recorded in the
+[`PageIndex adversarial fixture report`](doc/page_index_adversarial_fixture_report.md).
+
 For an adoption path that starts with parser-grounded source units and later
 adds LLM-Wiki cross-document maintenance, see
 [`doc/progressive_adoption_guide.md`](doc/progressive_adoption_guide.md). The

{graph_knowledge_doc_parser-0.2.2 → graph_knowledge_doc_parser-0.2.4}/README.md

@@ -32,6 +32,19 @@ The reusable helpers live under `src/workflow_ingest/` and are designed so the
 same core logic can be called from tests, scripts, and higher-level workflow
 code without duplicating orchestration.
 
+Layerwise parsing selects a strategy independently for each frontier layer.
+The default deterministic order is `layer_excerpt`, `layer_boundary`, then
+`page_index`; callers may provide another complete order, and optional triage
+may choose among the currently enabled strategies. Failed strategies are
+disabled for that layer and the workflow routes through the remaining methods
+before reaching explicit parse failure. PageIndex is a one-layer structural
+fallback that preserves exact source pointers and returns expandable children
+to normal strategy selection. See the
+[`0.2.4` release note](doc/release_0.2.4.md) and the
+[progressive refinement ADR](doc/adr_progressive_refinement_strategy_arbitration.md).
+The deterministic and text-only Bonsai fixture evaluation is recorded in the
+[`PageIndex adversarial fixture report`](doc/page_index_adversarial_fixture_report.md).
+
 For an adoption path that starts with parser-grounded source units and later
 adds LLM-Wiki cross-document maintenance, see
 [`doc/progressive_adoption_guide.md`](doc/progressive_adoption_guide.md). The

{graph_knowledge_doc_parser-0.2.2 → graph_knowledge_doc_parser-0.2.4}/kg_doc_parser/workflow_ingest/__init__.py

@@ -4,71 +4,34 @@ from .adapters import (
     build_parser_source_map,
     normalize_ocr_pages,
 )
+from .cache import WorkflowLLMCallCache
 from .clients import (
     CanonicalGraphPersistenceClient,
-    DocumentTreeApiPersistenceClient,
     DirectRuntimeIngestClient,
+    DocumentTreeApiPersistenceClient,
     IngestExecutionClient,
     ServerCanonicalKgClient,
     UnsupportedClientOperation,
 )
-from .cache import WorkflowLLMCallCache
-from .design import DEFAULT_WORKFLOW_ID, build_ingest_workflow_design, ensure_ingest_workflow_design
 from .demo_harness import DemoHarnessArtifacts, DemoHarnessConfig, run_demo_harness
-from .handlers import build_ingest_step_resolver
-from .parsing import (
-    OCRParseRequest,
-    PageIndexParseRequest,
-    ParseMode,
-    TreeParseRequest,
-    parse_document,
-    parse_ocr_document,
-    parse_page_index_document,
-    parse_tree_document,
-)
-from .ocr_pipeline import (
-    OCRImagePayload,
-    OCRWorkflowArtifacts,
-    OCRWorkflowStateStore,
-    prepare_ocr_workflow_input,
-    run_ocr_ingest_workflow,
-)
-from .runners import (
-    LayerwiseWorkflowCommandResult,
-    OcrWorkflowCommandResult,
-    PageIndexWorkflowCommandResult,
-    WorkflowCommandResult,
-    discover_input_files,
-    run_demo_harness_workflow,
-    run_layerwise_batch_workflow,
-    run_layerwise_source_workflow,
-    run_ocr_batch_workflow,
-    run_ocr_source_workflow,
-    run_page_index_batch_workflow,
-    run_page_index_source_workflow,
-)
-from .smoke_assets import generate_ocr_smoke_assets
-from .page_index import (
-    BlockAssignment,
-    BlockAssignmentBatch,
-    CandidateBlock,
-    PageIndexBlockSpec,
-    PageIndexParseResult,
-    PageIndexValidationResult,
-    build_page_index_workflow_input,
+from .design import (
+    DEFAULT_WORKFLOW_ID,
+    build_ingest_workflow_design,
+    ensure_ingest_workflow_design,
 )
+from .handlers import build_ingest_step_resolver
 from .models import (
     BoundingBox,
     CanonicalGraphWriteResult,
     CurrentLayerContext,
-    CurrentLayerReview,
     CurrentLayerResult,
-    LayerCoverageGap,
-    LayerDuplicateChildNote,
+    CurrentLayerReview,
     GroundedSourceRecord,
     IngestRunHandle,
     IngestRunResult,
     LayerChildCandidate,
+    LayerCoverageGap,
+    LayerDuplicateChildNote,
     LayerFrontierItem,
     LayerSpanConflict,
     NormalizedPage,
@@ -79,16 +42,23 @@ from .models import (
     WorkflowExportBundle,
     WorkflowIngestInput,
 )
-from .providers import (
-    EmbeddingProviderConfig,
-    FakeChatModel,
-    ProviderEndpointConfig,
-    WorkflowProviderSettings,
-    build_chat_model,
-    build_chat_model_for_role,
-    build_embedding_function,
+from .ocr_pipeline import (
+    OCRImagePayload,
+    OCRWorkflowArtifacts,
+    OCRWorkflowStateStore,
+    prepare_ocr_workflow_input,
+    run_ocr_ingest_workflow,
+)
+from .page_index import (
+    BlockAssignment,
+    BlockAssignmentBatch,
+    CandidateBlock,
+    PageIndexBlockSpec,
+    PageIndexParseResult,
+    PageIndexValidationResult,
+    build_page_index_workflow_input,
+    parse_page_index_layer,
 )
-from .probe import WorkflowProbe, emit_probe_event
 from .parser_core import (
     apply_cud_update,
     check_layer_coverage,
@@ -99,101 +69,157 @@ from .parser_core import (
     propose_layer_breakdown,
     review_layer,
 )
-from .service import build_default_engines, build_runtime, run_ingest_workflow
+from .parsing import (
+    OCRParseRequest,
+    PageIndexParseRequest,
+    ParseMode,
+    TreeParseRequest,
+    parse_document,
+    parse_ocr_document,
+    parse_page_index_document,
+    parse_tree_document,
+)
+from .probe import WorkflowProbe, emit_probe_event
+from .providers import (
+    EmbeddingProviderConfig,
+    FakeChatModel,
+    ProviderEndpointConfig,
+    WorkflowProviderSettings,
+    build_chat_model,
+    build_chat_model_for_role,
+    build_embedding_function,
+)
+from .runners import (
+    LayerwiseWorkflowCommandResult,
+    OcrWorkflowCommandResult,
+    PageIndexWorkflowCommandResult,
+    WorkflowCommandResult,
+    discover_input_files,
+    run_demo_harness_workflow,
+    run_layerwise_batch_workflow,
+    run_layerwise_source_workflow,
+    run_ocr_batch_workflow,
+    run_ocr_source_workflow,
+    run_page_index_batch_workflow,
+    run_page_index_source_workflow,
+)
 from .semantics import HydratedTextPointer, SemanticNode
+from .service import build_default_engines, build_runtime, run_ingest_workflow
+from .smoke_assets import generate_ocr_smoke_assets
+from .strategy import (
+    HARD_CODED_STRATEGY_PRIORITY,
+    ParseStrategy,
+    ParseStrategyAssessment,
+    ParseStrategyDecision,
+    ParseStrategyRequest,
+    ParseStrategyTriage,
+    build_llm_strategy_triage,
+    hardcoded_strategy,
+    select_parse_strategy,
+)
 
 __all__ = [
+    "DEFAULT_WORKFLOW_ID",
+    "HARD_CODED_STRATEGY_PRIORITY",
+    "BlockAssignment",
+    "BlockAssignmentBatch",
     "BoundingBox",
+    "CandidateBlock",
     "CanonicalGraphPersistenceClient",
     "CanonicalGraphWriteResult",
     "CurrentLayerContext",
-    "CurrentLayerReview",
     "CurrentLayerResult",
-    "LayerCoverageGap",
-    "LayerDuplicateChildNote",
+    "CurrentLayerReview",
     "DemoHarnessArtifacts",
     "DemoHarnessConfig",
-    "DocumentTreeApiPersistenceClient",
     "DirectRuntimeIngestClient",
+    "DocumentTreeApiPersistenceClient",
     "EmbeddingProviderConfig",
+    "FakeChatModel",
     "GroundedSourceRecord",
     "HydratedTextPointer",
-    "FakeChatModel",
     "IngestExecutionClient",
     "IngestRunHandle",
     "IngestRunResult",
     "LayerChildCandidate",
+    "LayerCoverageGap",
+    "LayerDuplicateChildNote",
     "LayerFrontierItem",
     "LayerSpanConflict",
+    "LayerwiseWorkflowCommandResult",
     "NormalizedPage",
     "NormalizedSourceCollection",
     "OCRImagePayload",
+    "OCRParseRequest",
     "OCRWorkflowArtifacts",
     "OCRWorkflowStateStore",
-    "OCRParseRequest",
-    "ParseSessionState",
-    "ProviderEndpointConfig",
-    "PageIndexParseRequest",
+    "OcrWorkflowCommandResult",
     "PageIndexBlockSpec",
+    "PageIndexParseRequest",
     "PageIndexParseResult",
     "PageIndexValidationResult",
+    "PageIndexWorkflowCommandResult",
     "ParseMode",
-    "TreeParseRequest",
+    "ParseSessionState",
+    "ParseStrategy",
+    "ParseStrategyAssessment",
+    "ParseStrategyDecision",
+    "ParseStrategyRequest",
+    "ParseStrategyTriage",
+    "ProviderEndpointConfig",
     "SemanticNode",
     "ServerCanonicalKgClient",
     "SourceUnit",
+    "TreeParseRequest",
     "UnsupportedClientOperation",
     "ValidationReport",
+    "WorkflowCommandResult",
+    "WorkflowExportBundle",
+    "WorkflowIngestInput",
     "WorkflowLLMCallCache",
     "WorkflowProbe",
     "WorkflowProviderSettings",
-    "WorkflowExportBundle",
-    "WorkflowIngestInput",
-    "WorkflowCommandResult",
-    "OcrWorkflowCommandResult",
-    "PageIndexWorkflowCommandResult",
-    "LayerwiseWorkflowCommandResult",
-    "CandidateBlock",
-    "BlockAssignment",
-    "BlockAssignmentBatch",
-    "DEFAULT_WORKFLOW_ID",
-    "build_chat_model",
-    "build_chat_model_for_role",
-    "build_embedding_function",
     "apply_cud_update",
-    "check_layer_coverage",
-    "default_parse_semantic_fn",
-    "finalize_semantic_tree",
-    "initialize_parse_session",
-    "prepare_layer_frontier",
-    "propose_layer_breakdown",
-    "review_layer",
     "build_authoritative_source_map",
+    "build_chat_model",
+    "build_chat_model_for_role",
     "build_default_engines",
-    "build_ingest_workflow_design",
+    "build_embedding_function",
     "build_ingest_step_resolver",
-    "build_parser_input_dict",
+    "build_ingest_workflow_design",
+    "build_llm_strategy_triage",
     "build_page_index_workflow_input",
+    "build_parser_input_dict",
     "build_parser_source_map",
     "build_runtime",
-    "ensure_ingest_workflow_design",
+    "check_layer_coverage",
+    "default_parse_semantic_fn",
+    "discover_input_files",
     "emit_probe_event",
+    "ensure_ingest_workflow_design",
+    "finalize_semantic_tree",
+    "generate_ocr_smoke_assets",
+    "hardcoded_strategy",
+    "initialize_parse_session",
     "normalize_ocr_pages",
     "parse_document",
     "parse_ocr_document",
     "parse_page_index_document",
+    "parse_page_index_layer",
     "parse_tree_document",
+    "prepare_layer_frontier",
     "prepare_ocr_workflow_input",
+    "propose_layer_breakdown",
+    "review_layer",
     "run_demo_harness",
     "run_demo_harness_workflow",
     "run_ingest_workflow",
-    "discover_input_files",
     "run_layerwise_batch_workflow",
     "run_layerwise_source_workflow",
     "run_ocr_batch_workflow",
+    "run_ocr_ingest_workflow",
     "run_ocr_source_workflow",
     "run_page_index_batch_workflow",
     "run_page_index_source_workflow",
-    "generate_ocr_smoke_assets",
-    "run_ocr_ingest_workflow",
+    "select_parse_strategy",
 ]

{graph_knowledge_doc_parser-0.2.2 → graph_knowledge_doc_parser-0.2.4}/kg_doc_parser/workflow_ingest/cli.py

@@ -46,14 +46,40 @@ def _provider_settings_from_args(args: argparse.Namespace) -> WorkflowProviderSe
         "api_key_env": args.parser_api_key_env,
     }
     proposal_mode = getattr(args, "proposal_mode", None)
+    parse_strategy = getattr(args, "parse_strategy", None)
+    parse_strategy_order = getattr(args, "parse_strategy_order", None)
+    triage_enabled = getattr(args, "triage_enabled", None)
+    page_index_summary_enabled = getattr(args, "page_index_summary_enabled", None)
     ocr_override = any(value is not None for value in ocr_values.values())
     parser_override = any(value is not None for value in parser_values.values())
     proposal_override = proposal_mode is not None
-    if not ocr_override and not parser_override and not proposal_override:
+    strategy_override = parse_strategy is not None
+    strategy_order_override = parse_strategy_order is not None
+    triage_override = triage_enabled is not None
+    summary_override = page_index_summary_enabled is not None
+    if (
+        not ocr_override
+        and not parser_override
+        and not proposal_override
+        and not strategy_override
+        and not strategy_order_override
+        and not triage_override
+        and not summary_override
+    ):
         return None
     settings = WorkflowProviderSettings.from_env()
     if proposal_override:
         settings = settings.model_copy(update={"proposal_mode": proposal_mode})
+    if strategy_override:
+        settings = settings.model_copy(update={"parse_strategy": parse_strategy})
+    if strategy_order_override:
+        settings = settings.model_copy(
+            update={"parse_strategy_order": tuple(parse_strategy_order.split(","))}
+        )
+    if triage_override:
+        settings = settings.model_copy(update={"triage_enabled": triage_enabled})
+    if summary_override:
+        settings = settings.model_copy(update={"page_index_summary_enabled": page_index_summary_enabled})
     if ocr_override:
         settings = settings.model_copy(
             update={
@@ -76,6 +102,29 @@ def _provider_settings_from_args(args: argparse.Namespace) -> WorkflowProviderSe
 def _add_provider_args(parser: argparse.ArgumentParser) -> None:
     group = parser.add_argument_group("provider overrides")
     group.add_argument("--proposal-mode", choices=["children", "boundaries"], default=None)
+    group.add_argument(
+        "--parse-strategy",
+        choices=["auto", "layer_excerpt", "layer_boundary", "page_index"],
+        default=None,
+        help="Select the parser strategy for every frontier layer in this parse call",
+    )
+    group.add_argument(
+        "--parse-strategy-order",
+        default=None,
+        help="comma-separated parser order, e.g. layer_boundary,layer_excerpt,page_index",
+    )
+    group.add_argument(
+        "--triage-enabled",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="Enable or disable provider-backed per-layer strategy triage",
+    )
+    group.add_argument(
+        "--page-index-summary-enabled",
+        action=argparse.BooleanOptionalAction,
+        default=None,
+        help="Enable or disable summaries on PageIndex nodes for this parse call",
+    )
     group.add_argument("--ocr-provider", default=None)
     group.add_argument("--ocr-model", default=None)
     group.add_argument("--ocr-base-url", default=None)

{graph_knowledge_doc_parser-0.2.2 → graph_knowledge_doc_parser-0.2.4}/kg_doc_parser/workflow_ingest/design.py

@@ -58,6 +58,8 @@ def _workflow_edge(
     edge_id: str,
     src: str,
     dst: str,
+    predicate: str | None = None,
+    is_default: bool = True,
 ) -> WorkflowEdge:
     dst_name = dst.split("|")[-1]
     return WorkflowEdge(
@@ -76,7 +78,8 @@ def _workflow_edge(
             "entity_type": "workflow_edge",
             "workflow_id": workflow_id,
             "wf_priority": 100,
-            "wf_is_default": True,
+            "wf_predicate": predicate,
+            "wf_is_default": is_default,
             "wf_multiplicity": "one",
             "wf_version": "v1",
         },
@@ -97,12 +100,14 @@ def build_ingest_workflow_design(
         ("init_parse_session", "init_parse_session", False, False),
         ("check_frontier_remaining", "check_frontier_remaining", False, False),
         ("prepare_layer_frontier", "prepare_layer_frontier", False, False),
-        ("propose_layer_breakdown", "propose_layer_breakdown", False, False),
+        ("triage_parse_strategy", "triage_parse_strategy", False, False),
+        ("layer_excerpt_method", "propose_layer_breakdown", False, False),
+        ("layer_boundary_method", "propose_layer_breakdown", False, False),
+        ("page_index_layer", "page_index_layer", False, False),
         ("review_cud_proposal", "review_cud_proposal", False, False),
         ("apply_cud_update", "apply_cud_update", False, False),
         ("check_layer_coverage", "check_layer_coverage", False, False),
         ("check_layer_satisfaction", "check_layer_satisfaction", False, False),
-        ("switch_split_strategy", "switch_split_strategy", False, False),
         ("repair_layer_pointers", "repair_layer_pointers", False, False),
         ("dedupe_and_filter_layer", "dedupe_and_filter_layer", False, False),
         ("commit_layer_children", "commit_layer_children", False, False),
@@ -112,6 +117,7 @@ def build_ingest_workflow_design(
         ("validate_tree", "validate_tree", False, False),
         ("export_graph", "export_graph", False, False),
         ("persist_canonical_graph", "persist_canonical_graph", False, False),
+        ("parse_failure", "parse_failure", False, True),
         ("end", "end", False, True),
     ]
     nodes = [
@@ -125,22 +131,26 @@ def build_ingest_workflow_design(
         for suffix, op, start, terminal in node_specs
     ]
     node_by_suffix = {node.safe_get_id().split("|")[-1]: node for node in nodes}
-    edge_pairs: Iterable[tuple[str, str]] = [
+    edge_pairs: Iterable[tuple[str, str] | tuple[str, str, str]] = [
         ("start", "normalize_input"),
         ("normalize_input", "build_source_map"),
         ("build_source_map", "init_parse_session"),
         ("init_parse_session", "check_frontier_remaining"),
         ("check_frontier_remaining", "prepare_layer_frontier"),
         ("check_frontier_remaining", "finalize_semantic_tree"),
-        ("prepare_layer_frontier", "propose_layer_breakdown"),
-        ("propose_layer_breakdown", "review_cud_proposal"),
+        ("prepare_layer_frontier", "triage_parse_strategy"),
+        ("triage_parse_strategy", "layer_excerpt_method", "parse_strategy_layer_excerpt"),
+        ("triage_parse_strategy", "layer_boundary_method", "parse_strategy_layer_boundary"),
+        ("triage_parse_strategy", "page_index_layer", "parse_strategy_page_index"),
+        ("triage_parse_strategy", "parse_failure", "strategy_selection_failed"),
+        ("layer_excerpt_method", "review_cud_proposal", "strategy_attempted"),
+        ("layer_boundary_method", "review_cud_proposal", "strategy_attempted"),
         ("review_cud_proposal", "apply_cud_update"),
         ("apply_cud_update", "check_layer_coverage"),
         ("check_layer_coverage", "check_layer_satisfaction"),
-        ("check_layer_satisfaction", "propose_layer_breakdown"),
-        ("check_layer_satisfaction", "switch_split_strategy"),
-        ("switch_split_strategy", "propose_layer_breakdown"),
-        ("check_layer_satisfaction", "repair_layer_pointers"),
+        ("check_layer_satisfaction", "repair_layer_pointers", "layer_satisfied"),
+        ("check_layer_satisfaction", "triage_parse_strategy", "strategy_failed_with_remaining"),
+        ("check_layer_satisfaction", "parse_failure", "all_strategies_exhausted"),
         ("repair_layer_pointers", "dedupe_and_filter_layer"),
         ("dedupe_and_filter_layer", "commit_layer_children"),
         ("commit_layer_children", "check_children_expandable"),
@@ -148,6 +158,7 @@ def build_ingest_workflow_design(
         ("check_children_expandable", "check_frontier_remaining"),
         ("enqueue_next_layer_frontier", "check_frontier_remaining"),
         ("finalize_semantic_tree", "validate_tree"),
+        ("page_index_layer", "review_cud_proposal", "strategy_attempted"),
         ("validate_tree", "export_graph"),
         ("export_graph", "persist_canonical_graph"),
         ("persist_canonical_graph", "end"),
@@ -155,11 +166,15 @@ def build_ingest_workflow_design(
     edges = [
         _workflow_edge(
             workflow_id=workflow_id,
-            edge_id=f"wf|{workflow_id}|e|{src}->{dst}",
+            edge_id=f"wf|{workflow_id}|e|{index}|{src}->{dst}",
             src=node_by_suffix[src].safe_get_id(),
             dst=node_by_suffix[dst].safe_get_id(),
+            predicate=predicate if len(edge_spec) == 3 else None,
+            is_default=len(edge_spec) == 2,
         )
-        for src, dst in edge_pairs
+        for index, edge_spec in enumerate(edge_pairs)
+        for src, dst, *predicate_values in [edge_spec]
+        for predicate in [predicate_values[0] if predicate_values else None]
     ]
     return nodes, edges