corp-extractor 0.5.0__py3-none-any.whl → 0.9.3__py3-none-any.whl

statement_extractor/pipeline/orchestrator.py

@@ -2,11 +2,11 @@
 ExtractionPipeline - Main orchestrator for the 5-stage extraction pipeline.
 
 Coordinates the flow of data through all pipeline stages:
-1. Splitting: Text → RawTriple
-2. Extraction: RawTriple → PipelineStatement
-3. Qualification: Entity → QualifiedEntity
-4. Canonicalization: QualifiedEntity → CanonicalEntity
-5. Labeling: Statement → LabeledStatement
+1. Splitting: Text → SplitSentence (atomic sentences)
+2. Extraction: SplitSentence → PipelineStatement (subject-predicate-object triples)
+3. Qualification: Entity → CanonicalEntity
+4. Labeling: Statement → LabeledStatement
+5. Taxonomy: Statement → TaxonomyResult
 """
 
 import logging
@@ -18,7 +18,6 @@ from .config import PipelineConfig, get_stage_name
 from .registry import PluginRegistry
 from ..models import (
     QualifiedEntity,
-    EntityQualifiers,
     CanonicalEntity,
     LabeledStatement,
     TaxonomyResult,
@@ -31,8 +30,12 @@ class ExtractionPipeline:
     """
     Main pipeline orchestrator.
 
-    Coordinates the flow of data through all 5 stages, invoking registered
-    plugins in priority order and accumulating results in PipelineContext.
+    Coordinates the flow of data through all 5 stages:
+    1. Splitting: Text → SplitSentence (using splitter plugins)
+    2. Extraction: SplitSentence → PipelineStatement (using extractor plugins)
+    3. Qualification: Entity → CanonicalEntity (using qualifier + canonicalizer plugins)
+    4. Labeling: Statement → LabeledStatement (using labeler plugins)
+    5. Taxonomy: Statement → TaxonomyResult (using taxonomy plugins)
     """
 
     def __init__(self, config: Optional[PipelineConfig] = None):
@@ -86,20 +89,16 @@ class ExtractionPipeline:
             if self.config.is_stage_enabled(2):
                 ctx = self._run_extraction(ctx)
 
-            # Stage 3: Qualification
+            # Stage 3: Qualification (runs qualifiers + canonicalizers)
             if self.config.is_stage_enabled(3):
                 ctx = self._run_qualification(ctx)
 
-            # Stage 4: Canonicalization
+            # Stage 4: Labeling
             if self.config.is_stage_enabled(4):
-                ctx = self._run_canonicalization(ctx)
-
-            # Stage 5: Labeling
-            if self.config.is_stage_enabled(5):
                 ctx = self._run_labeling(ctx)
 
-            # Stage 6: Taxonomy classification
-            if self.config.is_stage_enabled(6):
+            # Stage 5: Taxonomy classification
+            if self.config.is_stage_enabled(5):
                 ctx = self._run_taxonomy(ctx)
 
         except Exception as e:
@@ -116,7 +115,7 @@ class ExtractionPipeline:
         return ctx
 
     def _run_splitting(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 1: Split text into raw triples."""
+        """Stage 1: Split text into atomic sentences."""
         stage_name = get_stage_name(1)
         logger.debug(f"Running {stage_name} stage")
         start_time = time.time()
@@ -133,9 +132,9 @@ class ExtractionPipeline:
 
             logger.debug(f"Using splitter: {splitter.name}")
             try:
-                raw_triples = splitter.split(ctx.source_text, ctx)
-                ctx.raw_triples = raw_triples
-                logger.info(f"Splitting produced {len(raw_triples)} raw triples")
+                split_sentences = splitter.split(ctx.source_text, ctx)
+                ctx.split_sentences = split_sentences
+                logger.info(f"Splitting produced {len(split_sentences)} sentences")
                 break
             except Exception as e:
                 logger.exception(f"Splitter {splitter.name} failed")
@@ -147,13 +146,13 @@ class ExtractionPipeline:
         return ctx
 
     def _run_extraction(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 2: Extract statements with typed entities from raw triples."""
+        """Stage 2: Extract subject-predicate-object triples from split sentences."""
         stage_name = get_stage_name(2)
         logger.debug(f"Running {stage_name} stage")
         start_time = time.time()
 
-        if not ctx.raw_triples:
-            logger.debug("No raw triples to extract from")
+        if not ctx.split_sentences:
+            logger.debug("No split sentences to extract from")
             return ctx
 
         extractors = PluginRegistry.get_extractors()
@@ -178,7 +177,7 @@ class ExtractionPipeline:
 
             logger.debug(f"Using extractor: {extractor.name}")
             try:
-                statements = extractor.extract(ctx.raw_triples, ctx)
+                statements = extractor.extract(ctx.split_sentences, ctx)
                 ctx.statements = statements
                 logger.info(f"Extraction produced {len(statements)} statements")
                 break
@@ -211,7 +210,12 @@ class ExtractionPipeline:
         return schemas
 
     def _run_qualification(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 3: Add qualifiers to entities."""
+        """
+        Stage 3: Qualify entities with identifiers, canonical names, and FQNs.
+
+        Runs qualifier plugins for each entity type. Qualifier plugins now return
+        CanonicalEntity directly (with qualifiers, canonical match, and FQN).
+        """
         stage_name = get_stage_name(3)
         logger.debug(f"Running {stage_name} stage")
         start_time = time.time()
@@ -227,14 +231,15 @@ class ExtractionPipeline:
                 if entity.entity_ref not in entities_to_qualify:
                     entities_to_qualify[entity.entity_ref] = entity
 
-        logger.debug(f"Qualifying {len(entities_to_qualify)} unique entities")
+        logger.info(f"Stage 3: Qualifying {len(entities_to_qualify)} unique entities")
 
-        # Qualify each entity using applicable plugins
-        for entity_ref, entity in entities_to_qualify.items():
-            qualifiers = EntityQualifiers()
-            sources = []
+        # Process each entity through qualifier plugins
+        entities_list = list(entities_to_qualify.items())
+        for idx, (entity_ref, entity) in enumerate(entities_list, 1):
+            logger.info(f"  [{idx}/{len(entities_list)}] Qualifying '{entity.text}' ({entity.type.value})")
 
-            # Get qualifiers for this entity type
+            # Run qualifier plugins - first one to return a result wins
+            canonical = None
             type_qualifiers = PluginRegistry.get_qualifiers_for_type(entity.type)
 
             for qualifier_plugin in type_qualifiers:
@@ -242,86 +247,36 @@ class ExtractionPipeline:
                     continue
 
                 try:
-                    plugin_qualifiers = qualifier_plugin.qualify(entity, ctx)
-                    if plugin_qualifiers and plugin_qualifiers.has_any_qualifier():
-                        qualifiers = qualifiers.merge_with(plugin_qualifiers)
-                        sources.append(qualifier_plugin.name)
+                    result = qualifier_plugin.qualify(entity, ctx)
+                    if result is not None:
+                        canonical = result
+                        logger.info(f"    Qualified by {qualifier_plugin.name}: {canonical.fqn}")
+                        break  # Use first successful match
                 except Exception as e:
                     logger.error(f"Qualifier {qualifier_plugin.name} failed for {entity.text}: {e}")
                     ctx.add_error(f"Qualifier {qualifier_plugin.name} failed: {str(e)}")
                     if self.config.fail_fast:
                         raise
 
-            # Create QualifiedEntity
-            qualified = QualifiedEntity(
-                entity_ref=entity_ref,
-                original_text=entity.text,
-                entity_type=entity.type,
-                qualifiers=qualifiers,
-                qualification_sources=sources,
-            )
-            ctx.qualified_entities[entity_ref] = qualified
-
-        logger.info(f"Qualified {len(ctx.qualified_entities)} entities")
-        ctx.record_timing(stage_name, time.time() - start_time)
-        return ctx
-
-    def _run_canonicalization(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 4: Resolve entities to canonical forms."""
-        stage_name = get_stage_name(4)
-        logger.debug(f"Running {stage_name} stage")
-        start_time = time.time()
-
-        if not ctx.qualified_entities:
-            # Create basic qualified entities if stage 3 was skipped
-            for stmt in ctx.statements:
-                for entity in [stmt.subject, stmt.object]:
-                    if entity.entity_ref not in ctx.qualified_entities:
-                        ctx.qualified_entities[entity.entity_ref] = QualifiedEntity(
-                            entity_ref=entity.entity_ref,
-                            original_text=entity.text,
-                            entity_type=entity.type,
-                        )
-
-        # Canonicalize each qualified entity
-        for entity_ref, qualified in ctx.qualified_entities.items():
-            canonical_match = None
-            fqn = None
-
-            # Get canonicalizers for this entity type
-            type_canonicalizers = PluginRegistry.get_canonicalizers_for_type(qualified.entity_type)
-
-            for canon_plugin in type_canonicalizers:
-                if not self.config.is_plugin_enabled(canon_plugin.name):
-                    continue
-
-                try:
-                    match = canon_plugin.find_canonical(qualified, ctx)
-                    if match:
-                        canonical_match = match
-                        fqn = canon_plugin.format_fqn(qualified, match)
-                        break  # Use first successful match
-                except Exception as e:
-                    logger.error(f"Canonicalizer {canon_plugin.name} failed for {qualified.original_text}: {e}")
-                    ctx.add_error(f"Canonicalizer {canon_plugin.name} failed: {str(e)}")
-                    if self.config.fail_fast:
-                        raise
+            # Create fallback CanonicalEntity if no plugin matched
+            if canonical is None:
+                qualified = QualifiedEntity(
+                    entity_ref=entity_ref,
+                    original_text=entity.text,
+                    entity_type=entity.type,
+                )
+                canonical = CanonicalEntity.from_qualified(qualified=qualified)
+                logger.debug(f"    No qualification found, using original text")
 
-            # Create CanonicalEntity
-            canonical = CanonicalEntity.from_qualified(
-                qualified=qualified,
-                canonical_match=canonical_match,
-                fqn=fqn,
-            )
             ctx.canonical_entities[entity_ref] = canonical
 
-        logger.info(f"Canonicalized {len(ctx.canonical_entities)} entities")
+        logger.info(f"Qualified {len(ctx.canonical_entities)} entities")
         ctx.record_timing(stage_name, time.time() - start_time)
         return ctx
 
     def _run_labeling(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 5: Apply labels to statements."""
-        stage_name = get_stage_name(5)
+        """Stage 4: Apply labels to statements."""
+        stage_name = get_stage_name(4)
         logger.debug(f"Running {stage_name} stage")
         start_time = time.time()
 
@@ -329,9 +284,9 @@ class ExtractionPipeline:
             logger.debug("No statements to label")
             return ctx
 
-        # Ensure canonical entities exist
+        # Ensure canonical entities exist (run qualification if skipped)
         if not ctx.canonical_entities:
-            self._run_canonicalization(ctx)
+            self._run_qualification(ctx)
 
         labelers = PluginRegistry.get_labelers()
 
@@ -393,8 +348,10 @@ class ExtractionPipeline:
         return ctx
 
     def _run_taxonomy(self, ctx: PipelineContext) -> PipelineContext:
-        """Stage 6: Classify statements against taxonomies."""
-        stage_name = get_stage_name(6)
+        """Stage 5: Classify statements against taxonomies."""
+        from ..plugins.base import PluginCapability
+
+        stage_name = get_stage_name(5)
         logger.debug(f"Running {stage_name} stage")
         start_time = time.time()
 
@@ -408,27 +365,38 @@ class ExtractionPipeline:
             return ctx
 
         total_results = 0
-        for labeled_stmt in ctx.labeled_statements:
-            stmt = labeled_stmt.statement
-            subj_canonical = labeled_stmt.subject_canonical
-            obj_canonical = labeled_stmt.object_canonical
-
-            # Apply all taxonomy classifiers
-            for classifier in taxonomy_classifiers:
-                if not self.config.is_plugin_enabled(classifier.name):
-                    continue
 
-                try:
-                    results = classifier.classify(stmt, subj_canonical, obj_canonical, ctx)
+        # Prepare batch items: list of (statement, subject_canonical, object_canonical)
+        batch_items = [
+            (labeled_stmt.statement, labeled_stmt.subject_canonical, labeled_stmt.object_canonical)
+            for labeled_stmt in ctx.labeled_statements
+        ]
+
+        # Apply all taxonomy classifiers
+        for classifier in taxonomy_classifiers:
+            if not self.config.is_plugin_enabled(classifier.name):
+                continue
+
+            try:
+                # Require batch processing capability
+                if PluginCapability.BATCH_PROCESSING not in classifier.capabilities:
+                    raise RuntimeError(
+                        f"Taxonomy classifier '{classifier.name}' does not support batch processing. "
+                        "Pipeline requires BATCH_PROCESSING capability for efficient GPU utilization."
+                    )
+
+                logger.debug(f"Using batch classification for {classifier.name} ({len(batch_items)} items)")
+                batch_results = classifier.classify_batch(batch_items, ctx)
+
+                # Apply results to each labeled statement
+                for labeled_stmt, results in zip(ctx.labeled_statements, batch_results):
                     if results:
-                        # Store taxonomy results in context (list of results per key)
+                        stmt = labeled_stmt.statement
                         key = (stmt.source_text, classifier.taxonomy_name)
                         if key not in ctx.taxonomy_results:
                             ctx.taxonomy_results[key] = []
                         ctx.taxonomy_results[key].extend(results)
                         total_results += len(results)
-
-                        # Also add to the labeled statement for easy access
                         labeled_stmt.taxonomy_results.extend(results)
 
                         for result in results:
@@ -436,11 +404,12 @@ class ExtractionPipeline:
                                 f"Taxonomy {classifier.name}: {result.full_label} "
                                 f"(confidence={result.confidence:.2f})"
                             )
-                except Exception as e:
-                    logger.error(f"Taxonomy classifier {classifier.name} failed: {e}")
-                    ctx.add_error(f"Taxonomy classifier {classifier.name} failed: {str(e)}")
-                    if self.config.fail_fast:
-                        raise
+
+            except Exception as e:
+                logger.error(f"Taxonomy classifier {classifier.name} failed: {e}")
+                ctx.add_error(f"Taxonomy classifier {classifier.name} failed: {str(e)}")
+                if self.config.fail_fast:
+                    raise
 
         logger.info(f"Taxonomy produced {total_results} labels across {len(ctx.taxonomy_results)} statement-taxonomy pairs")
         ctx.record_timing(stage_name, time.time() - start_time)

statement_extractor/pipeline/registry.py

@@ -14,9 +14,10 @@ if TYPE_CHECKING:
         BaseSplitterPlugin,
         BaseExtractorPlugin,
         BaseQualifierPlugin,
-        BaseCanonicalizerPlugin,
         BaseLabelerPlugin,
         BaseTaxonomyPlugin,
+        BaseScraperPlugin,
+        BasePDFParserPlugin,
     )
     from ..models import EntityType
 
@@ -37,13 +38,15 @@ class PluginRegistry:
     _splitters: list["BaseSplitterPlugin"] = []
     _extractors: list["BaseExtractorPlugin"] = []
     _qualifiers: list["BaseQualifierPlugin"] = []
-    _canonicalizers: list["BaseCanonicalizerPlugin"] = []
     _labelers: list["BaseLabelerPlugin"] = []
     _taxonomy_classifiers: list["BaseTaxonomyPlugin"] = []
 
+    # Content acquisition plugins
+    _scrapers: list["BaseScraperPlugin"] = []
+    _pdf_parsers: list["BasePDFParserPlugin"] = []
+
     # Index by entity type for quick lookup
     _qualifiers_by_type: dict["EntityType", list["BaseQualifierPlugin"]] = {}
-    _canonicalizers_by_type: dict["EntityType", list["BaseCanonicalizerPlugin"]] = {}
 
     # Index by name for CLI lookup
     _all_plugins: dict[str, "BasePlugin"] = {}
@@ -54,11 +57,11 @@ class PluginRegistry:
         cls._splitters = []
         cls._extractors = []
         cls._qualifiers = []
-        cls._canonicalizers = []
         cls._labelers = []
         cls._taxonomy_classifiers = []
+        cls._scrapers = []
+        cls._pdf_parsers = []
         cls._qualifiers_by_type = {}
-        cls._canonicalizers_by_type = {}
         cls._all_plugins = {}
 
     # =========================================================================
@@ -100,25 +103,6 @@ class PluginRegistry:
             f"(priority={plugin.priority}, types={[t.value for t in plugin.supported_entity_types]})"
         )
 
-    @classmethod
-    def register_canonicalizer(cls, plugin: "BaseCanonicalizerPlugin") -> None:
-        """Register a canonicalizer plugin."""
-        cls._canonicalizers.append(plugin)
-        cls._canonicalizers.sort(key=lambda p: p.priority)
-        cls._all_plugins[plugin.name] = plugin
-
-        # Index by entity type
-        for entity_type in plugin.supported_entity_types:
-            if entity_type not in cls._canonicalizers_by_type:
-                cls._canonicalizers_by_type[entity_type] = []
-            cls._canonicalizers_by_type[entity_type].append(plugin)
-            cls._canonicalizers_by_type[entity_type].sort(key=lambda p: p.priority)
-
-        logger.debug(
-            f"Registered canonicalizer: {plugin.name} "
-            f"(priority={plugin.priority}, types={[t.value for t in plugin.supported_entity_types]})"
-        )
-
     @classmethod
     def register_labeler(cls, plugin: "BaseLabelerPlugin") -> None:
         """Register a labeler plugin."""
@@ -135,6 +119,22 @@ class PluginRegistry:
         cls._all_plugins[plugin.name] = plugin
         logger.debug(f"Registered taxonomy: {plugin.name} (priority={plugin.priority})")
 
+    @classmethod
+    def register_scraper(cls, plugin: "BaseScraperPlugin") -> None:
+        """Register a scraper plugin."""
+        cls._scrapers.append(plugin)
+        cls._scrapers.sort(key=lambda p: p.priority)
+        cls._all_plugins[plugin.name] = plugin
+        logger.debug(f"Registered scraper: {plugin.name} (priority={plugin.priority})")
+
+    @classmethod
+    def register_pdf_parser(cls, plugin: "BasePDFParserPlugin") -> None:
+        """Register a PDF parser plugin."""
+        cls._pdf_parsers.append(plugin)
+        cls._pdf_parsers.sort(key=lambda p: p.priority)
+        cls._all_plugins[plugin.name] = plugin
+        logger.debug(f"Registered PDF parser: {plugin.name} (priority={plugin.priority})")
+
     # =========================================================================
     # Decorator registration
     # =========================================================================
@@ -157,12 +157,6 @@ class PluginRegistry:
         cls.register_qualifier(plugin_class())
         return plugin_class
 
-    @classmethod
-    def canonicalizer(cls, plugin_class: Type[T]) -> Type[T]:
-        """Decorator to register a canonicalizer plugin class."""
-        cls.register_canonicalizer(plugin_class())
-        return plugin_class
-
     @classmethod
     def labeler(cls, plugin_class: Type[T]) -> Type[T]:
         """Decorator to register a labeler plugin class."""
@@ -175,6 +169,18 @@ class PluginRegistry:
         cls.register_taxonomy(plugin_class())
         return plugin_class
 
+    @classmethod
+    def scraper(cls, plugin_class: Type[T]) -> Type[T]:
+        """Decorator to register a scraper plugin class."""
+        cls.register_scraper(plugin_class())
+        return plugin_class
+
+    @classmethod
+    def pdf_parser(cls, plugin_class: Type[T]) -> Type[T]:
+        """Decorator to register a PDF parser plugin class."""
+        cls.register_pdf_parser(plugin_class())
+        return plugin_class
+
     # =========================================================================
     # Retrieval methods
     # =========================================================================
@@ -199,16 +205,6 @@ class PluginRegistry:
         """Get qualifier plugins that support a specific entity type."""
         return cls._qualifiers_by_type.get(entity_type, []).copy()
 
-    @classmethod
-    def get_canonicalizers(cls) -> list["BaseCanonicalizerPlugin"]:
-        """Get all registered canonicalizer plugins (sorted by priority)."""
-        return cls._canonicalizers.copy()
-
-    @classmethod
-    def get_canonicalizers_for_type(cls, entity_type: "EntityType") -> list["BaseCanonicalizerPlugin"]:
-        """Get canonicalizer plugins that support a specific entity type."""
-        return cls._canonicalizers_by_type.get(entity_type, []).copy()
-
     @classmethod
     def get_labelers(cls) -> list["BaseLabelerPlugin"]:
         """Get all registered labeler plugins (sorted by priority)."""
@@ -219,6 +215,16 @@ class PluginRegistry:
         """Get all registered taxonomy classifier plugins (sorted by priority)."""
         return cls._taxonomy_classifiers.copy()
 
+    @classmethod
+    def get_scrapers(cls) -> list["BaseScraperPlugin"]:
+        """Get all registered scraper plugins (sorted by priority)."""
+        return cls._scrapers.copy()
+
+    @classmethod
+    def get_pdf_parsers(cls) -> list["BasePDFParserPlugin"]:
+        """Get all registered PDF parser plugins (sorted by priority)."""
+        return cls._pdf_parsers.copy()
+
     @classmethod
     def get_plugin(cls, name: str) -> "BasePlugin | None":
         """Get a plugin by name."""
@@ -239,10 +245,8 @@ class PluginRegistry:
         elif stage == 3:
             return cls._qualifiers.copy()
         elif stage == 4:
-            return cls._canonicalizers.copy()
-        elif stage == 5:
             return cls._labelers.copy()
-        elif stage == 6:
+        elif stage == 5:
             return cls._taxonomy_classifiers.copy()
         return []
 
@@ -267,9 +271,11 @@ class PluginRegistry:
             (1, "splitting", cls._splitters),
             (2, "extraction", cls._extractors),
             (3, "qualification", cls._qualifiers),
-            (4, "canonicalization", cls._canonicalizers),
-            (5, "labeling", cls._labelers),
-            (6, "taxonomy", cls._taxonomy_classifiers),
+            (4, "labeling", cls._labelers),
+            (5, "taxonomy", cls._taxonomy_classifiers),
+            # Content acquisition plugins (stage 0)
+            (0, "scraper", cls._scrapers),
+            (-1, "pdf_parser", cls._pdf_parsers),
         ]
 
         for stage_num, stage_name, plugins in plugins_by_stage:

statement_extractor/plugins/__init__.py

@@ -4,10 +4,9 @@ Plugins module for the extraction pipeline.
 Contains all plugin implementations organized by stage:
 - splitters/: Stage 1 - Text to atomic triples
 - extractors/: Stage 2 - Refine entities and relations
-- qualifiers/: Stage 3 - Add qualifiers and identifiers
-- canonicalizers/: Stage 4 - Resolve canonical forms
-- labelers/: Stage 5 - Classify statements
-- taxonomy/: Stage 6 - Taxonomy classification
+- qualifiers/: Stage 3 - Qualify entities (add identifiers, canonical names, FQN)
+- labelers/: Stage 4 - Classify statements
+- taxonomy/: Stage 5 - Taxonomy classification
 """
 
 from .base import (
@@ -16,13 +15,20 @@ from .base import (
     BaseSplitterPlugin,
     BaseExtractorPlugin,
     BaseQualifierPlugin,
-    BaseCanonicalizerPlugin,
     BaseLabelerPlugin,
     BaseTaxonomyPlugin,
+    # Content acquisition plugins
+    ContentType,
+    ScraperResult,
+    PDFParseResult,
+    BaseScraperPlugin,
+    BasePDFParserPlugin,
 )
 
 # Import plugin modules for auto-registration
-from . import splitters, extractors, qualifiers, canonicalizers, labelers, taxonomy
+from . import splitters, extractors, qualifiers, labelers, taxonomy
+# Content acquisition plugins
+from . import scrapers, pdf
 
 __all__ = [
     "PluginCapability",
@@ -30,14 +36,20 @@ __all__ = [
     "BaseSplitterPlugin",
     "BaseExtractorPlugin",
     "BaseQualifierPlugin",
-    "BaseCanonicalizerPlugin",
     "BaseLabelerPlugin",
     "BaseTaxonomyPlugin",
+    # Content acquisition plugins
+    "ContentType",
+    "ScraperResult",
+    "PDFParseResult",
+    "BaseScraperPlugin",
+    "BasePDFParserPlugin",
     # Plugin modules
     "splitters",
     "extractors",
     "qualifiers",
-    "canonicalizers",
     "labelers",
     "taxonomy",
+    "scrapers",
+    "pdf",
 ]