corp-extractor 0.2.7__py3-none-any.whl

corp_extractor-0.2.7.dist-info/METADATA

@@ -0,0 +1,377 @@
+Metadata-Version: 2.4
+Name: corp-extractor
+Version: 0.2.7
+Summary: Extract structured statements from text using T5-Gemma 2 and Diverse Beam Search
+Project-URL: Homepage, https://github.com/corp-o-rate/statement-extractor
+Project-URL: Documentation, https://github.com/corp-o-rate/statement-extractor#readme
+Project-URL: Repository, https://github.com/corp-o-rate/statement-extractor
+Project-URL: Issues, https://github.com/corp-o-rate/statement-extractor/issues
+Author-email: Corp-o-Rate <neil@corp-o-rate.com>
+Maintainer-email: Corp-o-Rate <neil@corp-o-rate.com>
+License: MIT
+Keywords: diverse-beam-search,embeddings,gemma,information-extraction,knowledge-graph,nlp,statement-extraction,subject-predicate-object,t5,transformers,triples
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=5.0.0rc3
+Provides-Extra: all
+Requires-Dist: sentence-transformers>=2.2.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers>=2.2.0; extra == 'embeddings'
+Description-Content-Type: text/markdown
+
+# Corp Extractor
+
+Extract structured subject-predicate-object statements from unstructured text using the T5-Gemma 2 model.
+
+[![PyPI version](https://img.shields.io/pypi/v/corp-extractor.svg)](https://pypi.org/project/corp-extractor/)
+[![Python 3.10+](https://img.shields.io/pypi/pyversions/corp-extractor.svg)](https://pypi.org/project/corp-extractor/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- **Structured Extraction**: Converts unstructured text into subject-predicate-object triples
+- **Entity Type Recognition**: Identifies 12 entity types (ORG, PERSON, GPE, LOC, PRODUCT, EVENT, etc.)
+- **Quality Scoring** *(v0.2.0)*: Each triple scored for groundedness (0-1) based on source text
+- **Beam Merging** *(v0.2.0)*: Combines top beams for better coverage instead of picking one
+- **Embedding-based Dedup** *(v0.2.0)*: Uses semantic similarity to detect near-duplicate predicates
+- **Predicate Taxonomies** *(v0.2.0)*: Normalize predicates to canonical forms via embeddings
+- **Contextualized Matching** *(v0.2.2)*: Compares full "Subject Predicate Object" against source text for better accuracy
+- **Entity Type Merging** *(v0.2.3)*: Automatically merges UNKNOWN entity types with specific types during deduplication
+- **Reversal Detection** *(v0.2.3)*: Detects and corrects subject-object reversals using embedding comparison
+- **Command Line Interface** *(v0.2.4)*: Full-featured CLI for terminal usage
+- **Multiple Output Formats**: Get results as Pydantic models, JSON, XML, or dictionaries
+
+## Installation
+
+```bash
+# Recommended: include embedding support for smart deduplication
+pip install "corp-extractor[embeddings]"
+
+# Minimal installation (no embedding features)
+pip install corp-extractor
+```
+
+**Note**: This package requires `transformers>=5.0.0` (pre-release) for T5-Gemma2 model support. Install with `--pre` flag if needed:
+```bash
+pip install --pre "corp-extractor[embeddings]"
+```
+
+**For GPU support**, install PyTorch with CUDA first:
+```bash
+pip install torch --index-url https://download.pytorch.org/whl/cu121
+pip install "corp-extractor[embeddings]"
+```
+
+## Quick Start
+
+```python
+from statement_extractor import extract_statements
+
+result = extract_statements("""
+    Apple Inc. announced the iPhone 15 at their September event.
+    Tim Cook presented the new features to customers worldwide.
+""")
+
+for stmt in result:
+    print(f"{stmt.subject.text} ({stmt.subject.type})")
+    print(f"  --[{stmt.predicate}]--> {stmt.object.text}")
+    print(f"  Confidence: {stmt.confidence_score:.2f}")  # NEW in v0.2.0
+```
+
+## Command Line Interface
+
+The library includes a CLI for quick extraction from the terminal.
+
+### Install Globally (Recommended)
+
+For best results, install globally first:
+
+```bash
+# Using uv (recommended)
+uv tool install "corp-extractor[embeddings]"
+
+# Using pipx
+pipx install "corp-extractor[embeddings]"
+
+# Using pip
+pip install "corp-extractor[embeddings]"
+
+# Then use anywhere
+corp-extractor "Your text here"
+```
+
+### Quick Run with uvx
+
+Run directly without installing using [uv](https://docs.astral.sh/uv/):
+
+```bash
+uvx corp-extractor "Apple announced a new iPhone."
+```
+
+**Note**: First run downloads the model (~1.5GB) which may take a few minutes.
+
+### Usage Examples
+
+```bash
+# Extract from text argument
+corp-extractor "Apple Inc. announced the iPhone 15 at their September event."
+
+# Extract from file
+corp-extractor -f article.txt
+
+# Pipe from stdin
+cat article.txt | corp-extractor -
+
+# Output as JSON
+corp-extractor "Tim Cook is CEO of Apple." --json
+
+# Output as XML
+corp-extractor -f article.txt --xml
+
+# Verbose output with confidence scores
+corp-extractor -f article.txt --verbose
+
+# Use more beams for better quality
+corp-extractor -f article.txt --beams 8
+
+# Use custom predicate taxonomy
+corp-extractor -f article.txt --taxonomy predicates.txt
+
+# Use GPU explicitly
+corp-extractor -f article.txt --device cuda
+```
+
+### CLI Options
+
+```
+Usage: corp-extractor [OPTIONS] [TEXT]
+
+Options:
+  -f, --file PATH              Read input from file
+  -o, --output [table|json|xml] Output format (default: table)
+  --json                       Output as JSON (shortcut)
+  --xml                        Output as XML (shortcut)
+  -b, --beams INTEGER          Number of beams (default: 4)
+  --diversity FLOAT            Diversity penalty (default: 1.0)
+  --max-tokens INTEGER         Max tokens to generate (default: 2048)
+  --no-dedup                   Disable deduplication
+  --no-embeddings              Disable embedding-based dedup (faster)
+  --no-merge                   Disable beam merging
+  --dedup-threshold FLOAT      Deduplication threshold (default: 0.65)
+  --min-confidence FLOAT       Min confidence filter (default: 0)
+  --taxonomy PATH              Load predicate taxonomy from file
+  --taxonomy-threshold FLOAT   Taxonomy matching threshold (default: 0.5)
+  --device [auto|cuda|cpu]     Device to use (default: auto)
+  -v, --verbose                Show confidence scores and metadata
+  -q, --quiet                  Suppress progress messages
+  --version                    Show version
+  --help                       Show this message
+```
+
+## New in v0.2.0: Quality Scoring & Beam Merging
+
+By default, the library now:
+- **Scores each triple** for groundedness based on whether entities appear in source text
+- **Merges top beams** instead of selecting one, improving coverage
+- **Uses embeddings** to detect semantically similar predicates ("bought" ≈ "acquired")
+
+```python
+from statement_extractor import ExtractionOptions, ScoringConfig
+
+# Precision mode - filter low-confidence triples
+scoring = ScoringConfig(min_confidence=0.7)
+options = ExtractionOptions(scoring_config=scoring)
+result = extract_statements(text, options)
+
+# Access confidence scores
+for stmt in result:
+    print(f"{stmt} (confidence: {stmt.confidence_score:.2f})")
+```
+
+## New in v0.2.0: Predicate Taxonomies
+
+Normalize predicates to canonical forms using embedding similarity:
+
+```python
+from statement_extractor import PredicateTaxonomy, ExtractionOptions
+
+taxonomy = PredicateTaxonomy(predicates=[
+    "acquired", "founded", "works_for", "announced",
+    "invested_in", "partnered_with"
+])
+
+options = ExtractionOptions(predicate_taxonomy=taxonomy)
+result = extract_statements(text, options)
+
+# "bought" -> "acquired" via embedding similarity
+for stmt in result:
+    if stmt.canonical_predicate:
+        print(f"{stmt.predicate} -> {stmt.canonical_predicate}")
+```
+
+## New in v0.2.2: Contextualized Matching
+
+Predicate canonicalization and deduplication now use **contextualized matching**:
+- Compares full "Subject Predicate Object" strings against source text
+- Better accuracy because predicates are evaluated in context
+- When duplicates are found, keeps the statement with the best match to source text
+
+This means "Apple bought Beats" vs "Apple acquired Beats" are compared holistically, not just "bought" vs "acquired".
+
+## New in v0.2.3: Entity Type Merging & Reversal Detection
+
+### Entity Type Merging
+
+When deduplicating statements, entity types are now automatically merged. If one statement has `UNKNOWN` type and a duplicate has a specific type (like `ORG` or `PERSON`), the specific type is preserved:
+
+```python
+# Before deduplication:
+# Statement 1: AtlasBio Labs (UNKNOWN) --sued by--> CuraPharm (ORG)
+# Statement 2: AtlasBio Labs (ORG) --sued by--> CuraPharm (ORG)
+
+# After deduplication:
+# Single statement: AtlasBio Labs (ORG) --sued by--> CuraPharm (ORG)
+```
+
+### Subject-Object Reversal Detection
+
+The library now detects when subject and object may have been extracted in the wrong order by comparing embeddings against source text:
+
+```python
+from statement_extractor import PredicateComparer
+
+comparer = PredicateComparer()
+
+# Automatically detect and fix reversals
+fixed_statements = comparer.detect_and_fix_reversals(statements)
+
+for stmt in fixed_statements:
+    if stmt.was_reversed:
+        print(f"Fixed reversal: {stmt}")
+```
+
+**How it works:**
+1. For each statement with source text, compares:
+   - "Subject Predicate Object" embedding vs source text
+   - "Object Predicate Subject" embedding vs source text
+2. If the reversed form has higher similarity, swaps subject and object
+3. Sets `was_reversed=True` to indicate the correction
+
+During deduplication, reversed duplicates (e.g., "A -> P -> B" and "B -> P -> A") are now detected and merged, with the correct orientation determined by source text similarity.
+
+## Disable Embeddings (Faster, No Extra Dependencies)
+
+```python
+options = ExtractionOptions(
+    embedding_dedup=False,  # Use exact text matching
+    merge_beams=False,      # Select single best beam
+)
+result = extract_statements(text, options)
+```
+
+## Output Formats
+
+```python
+from statement_extractor import (
+    extract_statements,
+    extract_statements_as_json,
+    extract_statements_as_xml,
+    extract_statements_as_dict,
+)
+
+# Pydantic models (default)
+result = extract_statements(text)
+
+# JSON string
+json_output = extract_statements_as_json(text)
+
+# Raw XML (model's native format)
+xml_output = extract_statements_as_xml(text)
+
+# Python dictionary
+dict_output = extract_statements_as_dict(text)
+```
+
+## Batch Processing
+
+```python
+from statement_extractor import StatementExtractor
+
+extractor = StatementExtractor(device="cuda")  # or "cpu"
+
+texts = ["Text 1...", "Text 2...", "Text 3..."]
+for text in texts:
+    result = extractor.extract(text)
+    print(f"Found {len(result)} statements")
+```
+
+## Entity Types
+
+| Type | Description | Example |
+|------|-------------|---------|
+| `ORG` | Organizations | Apple Inc., United Nations |
+| `PERSON` | People | Tim Cook, Elon Musk |
+| `GPE` | Geopolitical entities | USA, California, Paris |
+| `LOC` | Non-GPE locations | Mount Everest, Pacific Ocean |
+| `PRODUCT` | Products | iPhone, Model S |
+| `EVENT` | Events | World Cup, CES 2024 |
+| `WORK_OF_ART` | Creative works | Mona Lisa, Game of Thrones |
+| `LAW` | Legal documents | GDPR, Clean Air Act |
+| `DATE` | Dates | 2024, January 15 |
+| `MONEY` | Monetary values | $50 million, €100 |
+| `PERCENT` | Percentages | 25%, 0.5% |
+| `QUANTITY` | Quantities | 500 employees, 1.5 tons |
+| `UNKNOWN` | Unrecognized | (fallback) |
+
+## How It Works
+
+This library uses the T5-Gemma 2 statement extraction model with **Diverse Beam Search** ([Vijayakumar et al., 2016](https://arxiv.org/abs/1610.02424)):
+
+1. **Diverse Beam Search**: Generates 4+ candidate outputs using beam groups with diversity penalty
+2. **Quality Scoring** *(v0.2.0)*: Each triple scored for groundedness in source text
+3. **Beam Merging** *(v0.2.0)*: Top beams combined for better coverage
+4. **Embedding Dedup** *(v0.2.0)*: Semantic similarity removes near-duplicate predicates
+5. **Predicate Normalization** *(v0.2.0)*: Optional taxonomy matching via embeddings
+6. **Contextualized Matching** *(v0.2.2)*: Full statement context used for canonicalization and dedup
+7. **Entity Type Merging** *(v0.2.3)*: UNKNOWN types merged with specific types during dedup
+8. **Reversal Detection** *(v0.2.3)*: Subject-object reversals detected and corrected via embedding comparison
+
+## Requirements
+
+- Python 3.10+
+- PyTorch 2.0+
+- Transformers 4.35+
+- Pydantic 2.0+
+- sentence-transformers 2.2+ *(optional, for embedding features)*
+- ~2GB VRAM (GPU) or ~4GB RAM (CPU)
+
+## Links
+
+- [Model on HuggingFace](https://huggingface.co/Corp-o-Rate-Community/statement-extractor)
+- [Web Demo](https://statement-extractor.corp-o-rate.com)
+- [Diverse Beam Search Paper](https://arxiv.org/abs/1610.02424)
+- [Corp-o-Rate](https://corp-o-rate.com)
+
+## License
+
+MIT License - see LICENSE file for details.

corp_extractor-0.2.7.dist-info/RECORD

@@ -0,0 +1,11 @@
+statement_extractor/__init__.py,sha256=MIZgn-lD9-XGJapzdyYxMhEJFRrTzftbRklrhwA4e8w,2967
+statement_extractor/canonicalization.py,sha256=ZMLs6RLWJa_rOJ8XZ7PoHFU13-zeJkOMDnvK-ZaFa5s,5991
+statement_extractor/cli.py,sha256=kJnZm_mbq4np1vTxSjczMZM5zGuDlC8Z5xLJd8O3xZ4,7605
+statement_extractor/extractor.py,sha256=PX0SiJnYUnh06seyH5W77FcPpcvLXwEM8IGsuVuRh0Q,22158
+statement_extractor/models.py,sha256=xDF3pDPhIiqiMwFMPV94aBEgZGbSe-x2TkshahOiCog,10739
+statement_extractor/predicate_comparer.py,sha256=iwBfNJFNOFv8ODKN9F9EtmknpCeSThOpnu6P_PJSmgE,24898
+statement_extractor/scoring.py,sha256=Wa1BW6jXtHD7dZkUXwdwE39hwFo2ko6BuIogBc4E2Lk,14493
+corp_extractor-0.2.7.dist-info/METADATA,sha256=ldfaiJqQEkCF8vlVq2H3t5MJOH9_C1QSAz80s33nSVU,13591
+corp_extractor-0.2.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+corp_extractor-0.2.7.dist-info/entry_points.txt,sha256=i0iKFqPIusvb-QTQ1zNnFgAqatgVah-jIhahbs5TToQ,115
+corp_extractor-0.2.7.dist-info/RECORD,,

corp_extractor-0.2.7.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

corp_extractor-0.2.7.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+corp-extractor = statement_extractor.cli:main
+statement-extractor = statement_extractor.cli:main

statement_extractor/__init__.py

@@ -0,0 +1,110 @@
+"""
+Statement Extractor - Extract structured statements from text using T5-Gemma 2.
+
+A Python library for extracting subject-predicate-object triples from unstructured text.
+Uses Diverse Beam Search (Vijayakumar et al., 2016) for high-quality extraction.
+
+Paper: https://arxiv.org/abs/1610.02424
+
+Features:
+- Quality-based beam scoring and merging
+- Embedding-based predicate comparison for smart deduplication
+- Configurable precision/recall tradeoff
+- Support for predicate taxonomies
+
+Example:
+    >>> from statement_extractor import extract_statements
+    >>> result = extract_statements("Apple Inc. announced a new iPhone today.")
+    >>> for stmt in result:
+    ...     print(f"{stmt.subject.text} -> {stmt.predicate} -> {stmt.object.text}")
+    Apple Inc. -> announced -> a new iPhone
+
+    >>> # Access confidence scores
+    >>> for stmt in result:
+    ...     print(f"{stmt} (confidence: {stmt.confidence_score:.2f})")
+
+    >>> # Get as different formats
+    >>> xml = extract_statements_as_xml("Some text...")
+    >>> json_str = extract_statements_as_json("Some text...")
+    >>> data = extract_statements_as_dict("Some text...")
+"""
+
+__version__ = "0.2.5"
+
+# Core models
+from .models import (
+    Entity,
+    EntityType,
+    ExtractionOptions,
+    ExtractionResult,
+    Statement,
+    # New in 0.2.0
+    PredicateMatch,
+    PredicateTaxonomy,
+    PredicateComparisonConfig,
+    ScoringConfig,
+)
+
+# Main extractor
+from .extractor import (
+    StatementExtractor,
+    extract_statements,
+    extract_statements_as_dict,
+    extract_statements_as_json,
+    extract_statements_as_xml,
+)
+
+# Canonicalization utilities
+from .canonicalization import (
+    Canonicalizer,
+    default_entity_canonicalizer,
+    deduplicate_statements_exact,
+)
+
+# Scoring utilities
+from .scoring import (
+    BeamScorer,
+    TripleScorer,
+)
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core models
+    "Entity",
+    "EntityType",
+    "ExtractionOptions",
+    "ExtractionResult",
+    "Statement",
+    # Configuration models (new in 0.2.0)
+    "PredicateMatch",
+    "PredicateTaxonomy",
+    "PredicateComparisonConfig",
+    "ScoringConfig",
+    # Extractor class
+    "StatementExtractor",
+    # Convenience functions
+    "extract_statements",
+    "extract_statements_as_dict",
+    "extract_statements_as_json",
+    "extract_statements_as_xml",
+    # Canonicalization
+    "Canonicalizer",
+    "default_entity_canonicalizer",
+    "deduplicate_statements_exact",
+    # Scoring
+    "BeamScorer",
+    "TripleScorer",
+]
+
+
+# Lazy imports for optional dependencies
+def __getattr__(name: str):
+    """Lazy import for optional modules."""
+    if name == "PredicateComparer":
+        from .predicate_comparer import PredicateComparer
+        return PredicateComparer
+    if name == "EmbeddingDependencyError":
+        from .predicate_comparer import EmbeddingDependencyError
+        return EmbeddingDependencyError
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")