corp-extractor 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

statement_extractor/cli.py

@@ -2,11 +2,13 @@
 Command-line interface for statement extraction.
 
 Usage:
-    corp-extractor "Your text here"
-    corp-extractor -f input.txt
-    cat input.txt | corp-extractor -
+    corp-extractor split "Your text here"
+    corp-extractor split -f input.txt
+    corp-extractor pipeline "Your text here" --stages 1-5
+    corp-extractor plugins list
 """
 
+import json
 import logging
 import sys
 from typing import Optional
@@ -34,10 +36,26 @@ def _configure_logging(verbose: bool) -> None:
         "statement_extractor.scoring",
         "statement_extractor.predicate_comparer",
         "statement_extractor.canonicalization",
-        "statement_extractor.spacy_extraction",
+        "statement_extractor.gliner_extraction",
+        "statement_extractor.pipeline",
+        "statement_extractor.plugins",
+        "statement_extractor.plugins.extractors.gliner2",
+        "statement_extractor.plugins.splitters",
+        "statement_extractor.plugins.labelers",
     ]:
         logging.getLogger(logger_name).setLevel(level)
 
+    # Suppress noisy third-party loggers
+    for noisy_logger in [
+        "httpcore.http11",
+        "httpcore.connection",
+        "httpx",
+        "urllib3",
+        "huggingface_hub",
+    ]:
+        logging.getLogger(noisy_logger).setLevel(logging.WARNING)
+
+
 from . import __version__
 from .models import (
     ExtractionOptions,
@@ -47,7 +65,33 @@ from .models import (
 )
 
 
-@click.command()
+@click.group()
+@click.version_option(version=__version__)
+def main():
+    """
+    Extract structured statements from text.
+
+    \b
+    Commands:
+        split      Extract sub-statements from text (simple, fast)
+        pipeline   Run the full 5-stage extraction pipeline
+        plugins    List or inspect available plugins
+
+    \b
+    Examples:
+        corp-extractor split "Apple announced a new iPhone."
+        corp-extractor split -f article.txt --json
+        corp-extractor pipeline "Apple CEO Tim Cook announced..." --stages 1-3
+        corp-extractor plugins list
+    """
+    pass
+
+
+# =============================================================================
+# Split command (simple extraction)
+# =============================================================================
+
+@main.command("split")
 @click.argument("text", required=False)
 @click.option("-f", "--file", "input_file", type=click.Path(exists=True), help="Read input from file")
 @click.option(
@@ -66,7 +110,8 @@ from .models import (
 @click.option("--no-dedup", is_flag=True, help="Disable deduplication")
 @click.option("--no-embeddings", is_flag=True, help="Disable embedding-based deduplication (faster)")
 @click.option("--no-merge", is_flag=True, help="Disable beam merging (select single best beam)")
-@click.option("--no-spacy", is_flag=True, help="Disable spaCy extraction (use raw model output)")
+@click.option("--no-gliner", is_flag=True, help="Disable GLiNER2 extraction (use raw model output)")
+@click.option("--predicates", type=str, help="Comma-separated list of predicate types for GLiNER2 relation extraction")
 @click.option("--all-triples", is_flag=True, help="Keep all candidate triples instead of selecting best per source")
 @click.option("--dedup-threshold", type=float, default=0.65, help="Similarity threshold for deduplication (default: 0.65)")
 # Quality options
@@ -79,8 +124,7 @@ from .models import (
 # Output options
 @click.option("-v", "--verbose", is_flag=True, help="Show verbose output with confidence scores")
 @click.option("-q", "--quiet", is_flag=True, help="Suppress progress messages")
-@click.version_option(version=__version__)
-def main(
+def split_cmd(
     text: Optional[str],
     input_file: Optional[str],
     output: str,
@@ -92,7 +136,8 @@ def main(
     no_dedup: bool,
     no_embeddings: bool,
     no_merge: bool,
-    no_spacy: bool,
+    no_gliner: bool,
+    predicates: Optional[str],
     all_triples: bool,
     dedup_threshold: float,
     min_confidence: float,
@@ -103,17 +148,18 @@ def main(
     quiet: bool,
 ):
     """
-    Extract structured statements from text.
+    Extract sub-statements from text using T5-Gemma model.
 
-    TEXT can be provided as an argument, read from a file with -f, or piped via stdin.
+    This command splits text into structured subject-predicate-object triples.
+    It's fast and simple - use 'pipeline' for full entity resolution.
 
     \b
     Examples:
-        corp-extractor "Apple announced a new iPhone."
-        corp-extractor -f article.txt --json
-        corp-extractor -f article.txt -o json --beams 8
-        cat article.txt | corp-extractor -
-        echo "Tim Cook is CEO of Apple." | corp-extractor - --verbose
+        corp-extractor split "Apple announced a new iPhone."
+        corp-extractor split -f article.txt --json
+        corp-extractor split -f article.txt -o json --beams 8
+        cat article.txt | corp-extractor split -
+        echo "Tim Cook is CEO of Apple." | corp-extractor split - --verbose
 
     \b
     Output formats:
@@ -133,10 +179,7 @@ def main(
     # Get input text
     input_text = _get_input_text(text, input_file)
     if not input_text:
-        raise click.UsageError(
-            "No input provided. Use: statement-extractor \"text\", "
-            "statement-extractor -f file.txt, or pipe via stdin."
-        )
+        raise click.UsageError("No input provided. Provide text argument or use -f file.txt")
 
     if not quiet:
         click.echo(f"Processing {len(input_text)} characters...", err=True)
@@ -157,6 +200,13 @@ def main(
     # Configure scoring
     scoring_config = ScoringConfig(min_confidence=min_confidence)
 
+    # Parse predicates if provided
+    predicate_list = None
+    if predicates:
+        predicate_list = [p.strip() for p in predicates.split(",") if p.strip()]
+        if not quiet:
+            click.echo(f"Using predicate list: {predicate_list}", err=True)
+
     # Configure extraction options
     options = ExtractionOptions(
         num_beams=beams,
@@ -165,7 +215,8 @@ def main(
         deduplicate=not no_dedup,
         embedding_dedup=not no_embeddings,
         merge_beams=not no_merge,
-        use_spacy_extraction=not no_spacy,
+        use_gliner_extraction=not no_gliner,
+        predicates=predicate_list,
         all_triples=all_triples,
         predicate_taxonomy=predicate_taxonomy,
         predicate_config=predicate_config,
@@ -200,6 +251,394 @@ def main(
         raise click.ClickException(f"Extraction failed: {e}")
 
 
+# =============================================================================
+# Pipeline command
+# =============================================================================
+
+@main.command("pipeline")
+@click.argument("text", required=False)
+@click.option("-f", "--file", "input_file", type=click.Path(exists=True), help="Read input from file")
+@click.option(
+    "--stages",
+    type=str,
+    default="1-6",
+    help="Stages to run (e.g., '1,2,3' or '1-3' or '1-6')"
+)
+@click.option(
+    "--skip-stages",
+    type=str,
+    default=None,
+    help="Stages to skip (e.g., '4,5')"
+)
+@click.option(
+    "--plugins",
+    "enabled_plugins",
+    type=str,
+    default=None,
+    help="Plugins to enable (comma-separated names)"
+)
+@click.option(
+    "--disable-plugins",
+    type=str,
+    default=None,
+    help="Plugins to disable (comma-separated names)"
+)
+@click.option(
+    "--no-default-predicates",
+    is_flag=True,
+    help="Disable default predicate taxonomy (GLiNER2 will only use entity extraction)"
+)
+@click.option(
+    "-o", "--output",
+    type=click.Choice(["table", "json", "yaml", "triples"], case_sensitive=False),
+    default="table",
+    help="Output format (default: table)"
+)
+@click.option("-v", "--verbose", is_flag=True, help="Show verbose output")
+@click.option("-q", "--quiet", is_flag=True, help="Suppress progress messages")
+def pipeline_cmd(
+    text: Optional[str],
+    input_file: Optional[str],
+    stages: str,
+    skip_stages: Optional[str],
+    enabled_plugins: Optional[str],
+    disable_plugins: Optional[str],
+    no_default_predicates: bool,
+    output: str,
+    verbose: bool,
+    quiet: bool,
+):
+    """
+    Run the full 5-stage extraction pipeline.
+
+    \b
+    Stages:
+        1. Splitting      - Text → Raw triples (T5-Gemma)
+        2. Extraction     - Raw triples → Typed statements (GLiNER2)
+        3. Qualification  - Add qualifiers and identifiers
+        4. Canonicalization - Resolve to canonical forms
+        5. Labeling       - Apply sentiment, relation type, confidence
+
+    \b
+    Examples:
+        corp-extractor pipeline "Apple CEO Tim Cook announced..."
+        corp-extractor pipeline -f article.txt --stages 1-3
+        corp-extractor pipeline "..." --plugins gleif,companies_house
+        corp-extractor pipeline "..." --disable-plugins sec_edgar
+    """
+    _configure_logging(verbose)
+
+    # Get input text
+    input_text = _get_input_text(text, input_file)
+    if not input_text:
+        raise click.UsageError("No input provided. Provide text argument or use -f file.txt")
+
+    if not quiet:
+        click.echo(f"Processing {len(input_text)} characters through pipeline...", err=True)
+
+    # Import pipeline components (also loads plugins)
+    from .pipeline import ExtractionPipeline, PipelineConfig
+    _load_all_plugins()
+
+    # Parse stages
+    enabled_stages = _parse_stages(stages)
+    if skip_stages:
+        skip_set = _parse_stages(skip_stages)
+        enabled_stages = enabled_stages - skip_set
+
+    if not quiet:
+        click.echo(f"Running stages: {sorted(enabled_stages)}", err=True)
+
+    # Parse plugin selection
+    enabled_plugin_set = None
+    if enabled_plugins:
+        enabled_plugin_set = {p.strip() for p in enabled_plugins.split(",") if p.strip()}
+
+    disabled_plugin_set = set()
+    if disable_plugins:
+        disabled_plugin_set = {p.strip() for p in disable_plugins.split(",") if p.strip()}
+
+    # Build extractor options
+    extractor_options = {}
+    if no_default_predicates:
+        extractor_options["use_default_predicates"] = False
+        if not quiet:
+            click.echo("Default predicates disabled - using entity extraction only", err=True)
+
+    # Create config
+    config = PipelineConfig(
+        enabled_stages=enabled_stages,
+        enabled_plugins=enabled_plugin_set,
+        disabled_plugins=disabled_plugin_set,
+        extractor_options=extractor_options,
+    )
+
+    # Run pipeline
+    try:
+        pipeline = ExtractionPipeline(config)
+        ctx = pipeline.process(input_text)
+
+        # Output results
+        if output == "json":
+            _print_pipeline_json(ctx)
+        elif output == "yaml":
+            _print_pipeline_yaml(ctx)
+        elif output == "triples":
+            _print_pipeline_triples(ctx)
+        else:
+            _print_pipeline_table(ctx, verbose)
+
+        # Report errors/warnings
+        if ctx.processing_errors and not quiet:
+            click.echo(f"\nErrors: {len(ctx.processing_errors)}", err=True)
+            for error in ctx.processing_errors:
+                click.echo(f"  - {error}", err=True)
+
+        if ctx.processing_warnings and verbose:
+            click.echo(f"\nWarnings: {len(ctx.processing_warnings)}", err=True)
+            for warning in ctx.processing_warnings:
+                click.echo(f"  - {warning}", err=True)
+
+    except Exception as e:
+        logging.exception("Pipeline error:")
+        raise click.ClickException(f"Pipeline failed: {e}")
+
+
+def _parse_stages(stages_str: str) -> set[int]:
+    """Parse stage string like '1,2,3' or '1-3' into a set of ints."""
+    result = set()
+    for part in stages_str.split(","):
+        part = part.strip()
+        if "-" in part:
+            start, end = part.split("-", 1)
+            for i in range(int(start), int(end) + 1):
+                result.add(i)
+        else:
+            result.add(int(part))
+    return result
+
+
+def _print_pipeline_json(ctx):
+    """Print pipeline results as JSON."""
+    output = {
+        "statement_count": ctx.statement_count,
+        "raw_triples": [t.model_dump() for t in ctx.raw_triples],
+        "statements": [s.model_dump() for s in ctx.statements],
+        "labeled_statements": [stmt.as_dict() for stmt in ctx.labeled_statements],
+        "timings": ctx.stage_timings,
+        "warnings": ctx.processing_warnings,
+        "errors": ctx.processing_errors,
+    }
+    click.echo(json.dumps(output, indent=2, default=str))
+
+
+def _print_pipeline_yaml(ctx):
+    """Print pipeline results as YAML."""
+    try:
+        import yaml
+        output = {
+            "statement_count": ctx.statement_count,
+            "statements": [stmt.as_dict() for stmt in ctx.labeled_statements],
+            "timings": ctx.stage_timings,
+        }
+        click.echo(yaml.dump(output, default_flow_style=False))
+    except ImportError:
+        click.echo("YAML output requires PyYAML: pip install pyyaml", err=True)
+        _print_pipeline_json(ctx)
+
+
+def _print_pipeline_triples(ctx):
+    """Print pipeline results as simple triples."""
+    if ctx.labeled_statements:
+        for stmt in ctx.labeled_statements:
+            click.echo(f"{stmt.subject_fqn}\t{stmt.statement.predicate}\t{stmt.object_fqn}")
+    elif ctx.statements:
+        for stmt in ctx.statements:
+            click.echo(f"{stmt.subject.text}\t{stmt.predicate}\t{stmt.object.text}")
+    elif ctx.raw_triples:
+        for triple in ctx.raw_triples:
+            click.echo(f"{triple.subject_text}\t{triple.predicate_text}\t{triple.object_text}")
+
+
+def _print_pipeline_table(ctx, verbose: bool):
+    """Print pipeline results in table format."""
+    # Try labeled statements first, then statements, then raw triples
+    if ctx.labeled_statements:
+        click.echo(f"\nExtracted {len(ctx.labeled_statements)} statement(s):\n")
+        click.echo("-" * 80)
+
+        for i, stmt in enumerate(ctx.labeled_statements, 1):
+            click.echo(f"{i}. {stmt.subject_fqn}")
+            click.echo(f"   --[{stmt.statement.predicate}]-->")
+            click.echo(f"   {stmt.object_fqn}")
+
+            # Show labels (always in recent versions, not just verbose)
+            for label in stmt.labels:
+                if isinstance(label.label_value, float):
+                    click.echo(f"   {label.label_type}: {label.label_value:.3f}")
+                else:
+                    click.echo(f"   {label.label_type}: {label.label_value}")
+
+            # Show top taxonomy results (sorted by confidence)
+            if stmt.taxonomy_results:
+                sorted_taxonomy = sorted(stmt.taxonomy_results, key=lambda t: t.confidence, reverse=True)
+                top_taxonomy = sorted_taxonomy[:5]  # Show top 5
+                taxonomy_strs = [f"{t.category}:{t.label} ({t.confidence:.2f})" for t in top_taxonomy]
+                click.echo(f"   topics: {', '.join(taxonomy_strs)}")
+                if len(sorted_taxonomy) > 5:
+                    click.echo(f"   ... and {len(sorted_taxonomy) - 5} more topics")
+
+            if verbose and stmt.statement.source_text:
+                source = stmt.statement.source_text[:60] + "..." if len(stmt.statement.source_text) > 60 else stmt.statement.source_text
+                click.echo(f"   Source: \"{source}\"")
+
+            click.echo("-" * 80)
+
+    elif ctx.statements:
+        click.echo(f"\nExtracted {len(ctx.statements)} statement(s):\n")
+        click.echo("-" * 80)
+
+        for i, stmt in enumerate(ctx.statements, 1):
+            subj_type = f" ({stmt.subject.type.value})" if stmt.subject.type.value != "UNKNOWN" else ""
+            obj_type = f" ({stmt.object.type.value})" if stmt.object.type.value != "UNKNOWN" else ""
+
+            click.echo(f"{i}. {stmt.subject.text}{subj_type}")
+            click.echo(f"   --[{stmt.predicate}]-->")
+            click.echo(f"   {stmt.object.text}{obj_type}")
+
+            if verbose and stmt.confidence_score is not None:
+                click.echo(f"   Confidence: {stmt.confidence_score:.2f}")
+
+            click.echo("-" * 80)
+
+    elif ctx.raw_triples:
+        click.echo(f"\nExtracted {len(ctx.raw_triples)} raw triple(s):\n")
+        click.echo("-" * 80)
+
+        for i, triple in enumerate(ctx.raw_triples, 1):
+            click.echo(f"{i}. {triple.subject_text}")
+            click.echo(f"   --[{triple.predicate_text}]-->")
+            click.echo(f"   {triple.object_text}")
+
+            if verbose:
+                click.echo(f"   Confidence: {triple.confidence:.2f}")
+                if triple.source_sentence:
+                    source = triple.source_sentence[:60] + "..." if len(triple.source_sentence) > 60 else triple.source_sentence
+                    click.echo(f"   Source: \"{source}\"")
+
+            click.echo("-" * 80)
+
+    else:
+        click.echo("No statements extracted.")
+        return
+
+    # Show timings in verbose mode
+    if verbose and ctx.stage_timings:
+        click.echo("\nStage timings:")
+        for stage, duration in ctx.stage_timings.items():
+            click.echo(f"  {stage}: {duration:.3f}s")
+
+
+# =============================================================================
+# Plugins command
+# =============================================================================
+
+@main.command("plugins")
+@click.argument("action", type=click.Choice(["list", "info"]))
+@click.argument("plugin_name", required=False)
+@click.option("--stage", type=int, help="Filter by stage number (1-5)")
+def plugins_cmd(action: str, plugin_name: Optional[str], stage: Optional[int]):
+    """
+    List or inspect available plugins.
+
+    \b
+    Actions:
+        list   List all available plugins
+        info   Show details about a specific plugin
+
+    \b
+    Examples:
+        corp-extractor plugins list
+        corp-extractor plugins list --stage 3
+        corp-extractor plugins info gleif_qualifier
+    """
+    # Import and load plugins
+    _load_all_plugins()
+
+    from .pipeline.registry import PluginRegistry
+
+    if action == "list":
+        plugins = PluginRegistry.list_plugins(stage=stage)
+        if not plugins:
+            click.echo("No plugins registered.")
+            return
+
+        # Group by stage
+        by_stage: dict[int, list] = {}
+        for plugin in plugins:
+            stage_num = plugin["stage"]
+            if stage_num not in by_stage:
+                by_stage[stage_num] = []
+            by_stage[stage_num].append(plugin)
+
+        for stage_num in sorted(by_stage.keys()):
+            stage_plugins = by_stage[stage_num]
+            stage_name = stage_plugins[0]["stage_name"]
+            click.echo(f"\nStage {stage_num}: {stage_name.title()}")
+            click.echo("-" * 40)
+
+            for p in stage_plugins:
+                entity_types = p.get("entity_types", [])
+                types_str = f" ({', '.join(entity_types)})" if entity_types else ""
+                click.echo(f"  {p['name']}{types_str}  [priority: {p['priority']}]")
+
+    elif action == "info":
+        if not plugin_name:
+            raise click.UsageError("Plugin name required for 'info' action")
+
+        plugin = PluginRegistry.get_plugin(plugin_name)
+        if not plugin:
+            raise click.ClickException(f"Plugin not found: {plugin_name}")
+
+        click.echo(f"\nPlugin: {plugin.name}")
+        click.echo(f"Priority: {plugin.priority}")
+        click.echo(f"Capabilities: {plugin.capabilities.name if plugin.capabilities else 'NONE'}")
+
+        if plugin.description:
+            click.echo(f"Description: {plugin.description}")
+
+        if hasattr(plugin, "supported_entity_types"):
+            types = [t.value for t in plugin.supported_entity_types]
+            click.echo(f"Entity types: {', '.join(types)}")
+
+        if hasattr(plugin, "label_type"):
+            click.echo(f"Label type: {plugin.label_type}")
+
+        if hasattr(plugin, "supported_identifier_types"):
+            ids = plugin.supported_identifier_types
+            if ids:
+                click.echo(f"Supported identifiers: {', '.join(ids)}")
+
+        if hasattr(plugin, "provided_identifier_types"):
+            ids = plugin.provided_identifier_types
+            if ids:
+                click.echo(f"Provided identifiers: {', '.join(ids)}")
+
+
+def _load_all_plugins():
+    """Load all plugins by importing their modules."""
+    # Import all plugin modules to trigger registration
+    try:
+        from .plugins import splitters, extractors, qualifiers, canonicalizers, labelers, taxonomy
+        # The @PluginRegistry decorators will register plugins on import
+    except ImportError as e:
+        logging.debug(f"Some plugins failed to load: {e}")
+
+
+# =============================================================================
+# Helper functions
+# =============================================================================
+
 def _get_input_text(text: Optional[str], input_file: Optional[str]) -> Optional[str]:
     """Get input text from argument, file, or stdin."""
     if text == "-" or (text is None and input_file is None and not sys.stdin.isatty()):