databridge-core 0.1.0__tar.gz

databridge_core-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,fuzzy]"
+
+      - name: Lint with ruff
+        run: ruff check src/ tests/
+
+      - name: Run tests
+        run: pytest -v --tb=short
+
+      - name: Build package
+        run: python -m build
+
+  build-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install and verify
+        run: |
+          pip install -e ".[fuzzy]"
+          python -c "from databridge_core import compare_hashes, profile_data, load_csv; print('OK')"
+          databridge --version
+          databridge --help

databridge_core-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.pyc
+*.pyo
+dist/
+build/
+*.egg-info/
+.eggs/
+.pytest_cache/
+.ruff_cache/
+*.egg
+.venv/
+venv/

databridge_core-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 DataBridge AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

databridge_core-0.1.0/PKG-INFO

@@ -0,0 +1,124 @@
+Metadata-Version: 2.4
+Name: databridge-core
+Version: 0.1.0
+Summary: Upload your Chart of Accounts. Get a production-ready financial hierarchy and dbt models. Zero config.
+Project-URL: Homepage, https://github.com/datanexum/databridge-core
+Project-URL: Documentation, https://github.com/datanexum/databridge-core#readme
+Project-URL: Repository, https://github.com/datanexum/databridge-core
+Project-URL: Issues, https://github.com/datanexum/databridge-core/issues
+Author-email: DataBridge AI <hello@databridgeai.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: csv,data,diff,etl,finance,fuzzy-match,profiling,reconciliation
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Office/Business :: Financial
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Requires-Dist: pandas>=1.5
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Provides-Extra: all
+Requires-Dist: pillow>=9.0; extra == 'all'
+Requires-Dist: pypdf>=3.0; extra == 'all'
+Requires-Dist: pytesseract>=0.3; extra == 'all'
+Requires-Dist: rapidfuzz>=3.0; extra == 'all'
+Requires-Dist: sqlalchemy>=2.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1; extra == 'dev'
+Provides-Extra: fuzzy
+Requires-Dist: rapidfuzz>=3.0; extra == 'fuzzy'
+Provides-Extra: ocr
+Requires-Dist: pillow>=9.0; extra == 'ocr'
+Requires-Dist: pytesseract>=0.3; extra == 'ocr'
+Provides-Extra: pdf
+Requires-Dist: pypdf>=3.0; extra == 'pdf'
+Provides-Extra: sql
+Requires-Dist: sqlalchemy>=2.0; extra == 'sql'
+Description-Content-Type: text/markdown
+
+# DataBridge Core
+
+**Your finance team just spent 4 hours on VLOOKUP. This takes 5 seconds.**
+
+DataBridge Core is a Python toolkit for data reconciliation, profiling, and ingestion. Compare CSV files, find fuzzy matches, detect schema drift, and clean messy data -- from the command line or Python.
+
+```bash
+pip install databridge-core
+```
+
+## 5-Second Demo
+
+```bash
+# Profile a file
+databridge profile sales.csv
+
+# Compare two sources -- find orphans, conflicts, match rate
+databridge compare source.csv target.csv --keys id
+
+# Fuzzy match names across systems
+databridge fuzzy erp_accounts.csv gl_accounts.csv --column name --threshold 80
+```
+
+## Python API
+
+```python
+from databridge_core import compare_hashes, profile_data, load_csv
+
+# Profile your data
+profile = profile_data("chart_of_accounts.csv")
+print(f"{profile['rows']} rows, {profile['columns']} columns")
+print(f"Potential keys: {profile['potential_key_columns']}")
+
+# Compare two sources
+result = compare_hashes("source.csv", "target.csv", key_columns="account_id")
+stats = result["statistics"]
+print(f"Match rate: {stats['match_rate_percent']}%")
+print(f"Conflicts: {stats['conflicts']}, Orphans: {stats['total_orphans']}")
+```
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `databridge profile <file>` | Profile data: structure, quality, cardinality |
+| `databridge compare <a> <b> --keys <col>` | Hash comparison: orphans, conflicts, match rate |
+| `databridge fuzzy <a> <b> -c <col>` | Fuzzy match columns across two files |
+| `databridge diff <a> <b>` | Text diff between two files |
+| `databridge drift <old> <new>` | Detect schema drift between CSVs |
+| `databridge transform <file> -c <col> --op upper` | Clean a column (upper/lower/strip/trim/remove_special) |
+| `databridge merge <a> <b> --keys <col>` | Merge two CSVs on key columns |
+| `databridge find "*.csv"` | Find files matching a pattern |
+| `databridge parse <text>` | Parse tabular data from messy text |
+
+## Optional Extras
+
+```bash
+pip install 'databridge-core[fuzzy]'   # Fuzzy matching (rapidfuzz)
+pip install 'databridge-core[pdf]'     # PDF text extraction (pypdf)
+pip install 'databridge-core[ocr]'     # OCR image extraction (pytesseract)
+pip install 'databridge-core[sql]'     # Database queries (sqlalchemy)
+pip install 'databridge-core[all]'     # Everything
+pip install 'databridge-core[dev]'     # Development tools (pytest, ruff, build)
+```
+
+## Built for Finance
+
+DataBridge Core is the open-source foundation of [DataBridge AI](https://github.com/datanexum/databridge-ai) -- a full platform for financial hierarchy management, dbt model generation, and enterprise data reconciliation.
+
+**How it works:** Upload your Chart of Accounts. Get a production-ready financial hierarchy and dbt models. Zero config.
+
+## License
+
+MIT

databridge_core-0.1.0/README.md

@@ -0,0 +1,74 @@
+# DataBridge Core
+
+**Your finance team just spent 4 hours on VLOOKUP. This takes 5 seconds.**
+
+DataBridge Core is a Python toolkit for data reconciliation, profiling, and ingestion. Compare CSV files, find fuzzy matches, detect schema drift, and clean messy data -- from the command line or Python.
+
+```bash
+pip install databridge-core
+```
+
+## 5-Second Demo
+
+```bash
+# Profile a file
+databridge profile sales.csv
+
+# Compare two sources -- find orphans, conflicts, match rate
+databridge compare source.csv target.csv --keys id
+
+# Fuzzy match names across systems
+databridge fuzzy erp_accounts.csv gl_accounts.csv --column name --threshold 80
+```
+
+## Python API
+
+```python
+from databridge_core import compare_hashes, profile_data, load_csv
+
+# Profile your data
+profile = profile_data("chart_of_accounts.csv")
+print(f"{profile['rows']} rows, {profile['columns']} columns")
+print(f"Potential keys: {profile['potential_key_columns']}")
+
+# Compare two sources
+result = compare_hashes("source.csv", "target.csv", key_columns="account_id")
+stats = result["statistics"]
+print(f"Match rate: {stats['match_rate_percent']}%")
+print(f"Conflicts: {stats['conflicts']}, Orphans: {stats['total_orphans']}")
+```
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `databridge profile <file>` | Profile data: structure, quality, cardinality |
+| `databridge compare <a> <b> --keys <col>` | Hash comparison: orphans, conflicts, match rate |
+| `databridge fuzzy <a> <b> -c <col>` | Fuzzy match columns across two files |
+| `databridge diff <a> <b>` | Text diff between two files |
+| `databridge drift <old> <new>` | Detect schema drift between CSVs |
+| `databridge transform <file> -c <col> --op upper` | Clean a column (upper/lower/strip/trim/remove_special) |
+| `databridge merge <a> <b> --keys <col>` | Merge two CSVs on key columns |
+| `databridge find "*.csv"` | Find files matching a pattern |
+| `databridge parse <text>` | Parse tabular data from messy text |
+
+## Optional Extras
+
+```bash
+pip install 'databridge-core[fuzzy]'   # Fuzzy matching (rapidfuzz)
+pip install 'databridge-core[pdf]'     # PDF text extraction (pypdf)
+pip install 'databridge-core[ocr]'     # OCR image extraction (pytesseract)
+pip install 'databridge-core[sql]'     # Database queries (sqlalchemy)
+pip install 'databridge-core[all]'     # Everything
+pip install 'databridge-core[dev]'     # Development tools (pytest, ruff, build)
+```
+
+## Built for Finance
+
+DataBridge Core is the open-source foundation of [DataBridge AI](https://github.com/datanexum/databridge-ai) -- a full platform for financial hierarchy management, dbt model generation, and enterprise data reconciliation.
+
+**How it works:** Upload your Chart of Accounts. Get a production-ready financial hierarchy and dbt models. Zero config.
+
+## License
+
+MIT

databridge_core-0.1.0/examples/customers_a.csv

@@ -0,0 +1,11 @@
+id,name,email,city,balance
+1,Alice Johnson,alice@example.com,New York,1500.00
+2,Bob Smith,bob@example.com,Chicago,2300.50
+3,Charlie Brown,charlie@example.com,Houston,850.75
+4,Diana Prince,diana@example.com,Phoenix,3200.00
+5,Eve Williams,eve@example.com,San Antonio,1100.25
+6,Frank Castle,frank@example.com,Dallas,4500.00
+7,Grace Hopper,grace@example.com,San Jose,2750.30
+8,Hank Pym,hank@example.com,Austin,990.00
+9,Ivy League,ivy@example.com,Columbus,1800.60
+10,Jack Ryan,jack@example.com,Charlotte,3100.45

databridge_core-0.1.0/examples/customers_b.csv

@@ -0,0 +1,10 @@
+id,name,email,city,balance
+1,Alice Johnson,alice@example.com,New York,1500.00
+2,Bob Smith,bob@example.com,Chicago,2400.50
+3,Charles Brown,charlie@example.com,Houston,850.75
+4,Diana Prince,diana@example.com,Scottsdale,3200.00
+5,Eve Williams,eve@example.com,San Antonio,1100.25
+6,Frank Castle,frank@example.com,Dallas,4500.00
+7,Grace Hopper,grace@example.com,San Jose,2750.30
+8,Hank Pym,hank@example.com,Austin,990.00
+11,Kate Bishop,kate@example.com,Denver,2100.00

databridge_core-0.1.0/examples/demo.py

@@ -0,0 +1,58 @@
+"""DataBridge Core -- Quick demo.
+
+Run: python examples/demo.py
+"""
+
+from pathlib import Path
+
+# Resolve example file paths
+examples_dir = Path(__file__).parent
+file_a = str(examples_dir / "customers_a.csv")
+file_b = str(examples_dir / "customers_b.csv")
+
+
+def main():
+    from databridge_core import compare_hashes, profile_data, load_csv
+
+    # 1. Profile the source file
+    print("=" * 60)
+    print("1. PROFILE SOURCE DATA")
+    print("=" * 60)
+    profile = profile_data(file_a)
+    print(f"  File: {profile['file']}")
+    print(f"  Rows: {profile['rows']}, Columns: {profile['columns']}")
+    print(f"  Type: {profile['structure_type']}")
+    print(f"  Potential keys: {profile['potential_key_columns']}")
+    print()
+
+    # 2. Compare two sources
+    print("=" * 60)
+    print("2. COMPARE SOURCES")
+    print("=" * 60)
+    result = compare_hashes(file_a, file_b, key_columns="id")
+    stats = result["statistics"]
+    print(f"  Source A: {result['source_a']['total_rows']} rows")
+    print(f"  Source B: {result['source_b']['total_rows']} rows")
+    print(f"  Exact matches: {stats['exact_matches']}")
+    print(f"  Conflicts: {stats['conflicts']}")
+    print(f"  Orphans in A: {stats['orphans_only_in_source_a']}")
+    print(f"  Orphans in B: {stats['orphans_only_in_source_b']}")
+    print(f"  Match rate: {stats['match_rate_percent']}%")
+    print()
+
+    # 3. Load and preview
+    print("=" * 60)
+    print("3. LOAD & PREVIEW")
+    print("=" * 60)
+    loaded = load_csv(file_a, preview_rows=3)
+    print(f"  Columns: {loaded['columns']}")
+    print(f"  Preview (first 3 rows):")
+    for row in loaded["preview"]:
+        print(f"    {row}")
+    print()
+
+    print("Done! Try the CLI: databridge profile examples/customers_a.csv")
+
+
+if __name__ == "__main__":
+    main()

databridge_core-0.1.0/pyproject.toml

@@ -0,0 +1,76 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "databridge-core"
+version = "0.1.0"
+description = "Upload your Chart of Accounts. Get a production-ready financial hierarchy and dbt models. Zero config."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "DataBridge AI", email = "hello@databridgeai.com" },
+]
+keywords = ["data", "reconciliation", "profiling", "etl", "finance", "csv", "diff", "fuzzy-match"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Financial and Insurance Industry",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Office/Business :: Financial",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+]
+
+dependencies = [
+    "pandas>=1.5",
+    "pydantic>=2.0",
+    "click>=8.0",
+    "rich>=13.0",
+]
+
+[project.optional-dependencies]
+fuzzy = ["rapidfuzz>=3.0"]
+pdf = ["pypdf>=3.0"]
+ocr = ["pytesseract>=0.3", "Pillow>=9.0"]
+sql = ["sqlalchemy>=2.0"]
+all = [
+    "rapidfuzz>=3.0",
+    "pypdf>=3.0",
+    "pytesseract>=0.3",
+    "Pillow>=9.0",
+    "sqlalchemy>=2.0",
+]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "ruff>=0.1",
+    "build>=1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/datanexum/databridge-core"
+Documentation = "https://github.com/datanexum/databridge-core#readme"
+Repository = "https://github.com/datanexum/databridge-core"
+Issues = "https://github.com/datanexum/databridge-core/issues"
+
+[project.scripts]
+databridge = "databridge_core.cli:cli"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/databridge_core"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

databridge_core-0.1.0/src/databridge_core/__init__.py

@@ -0,0 +1,72 @@
+"""DataBridge Core -- Data reconciliation, profiling, and ingestion toolkit.
+
+Upload your Chart of Accounts. Get a production-ready financial hierarchy
+and dbt models. Zero config.
+
+Quick start::
+
+    from databridge_core import compare_hashes, profile_data, load_csv
+
+    result = profile_data("sales.csv")
+    print(result["rows"], "rows,", result["columns"], "columns")
+
+    comparison = compare_hashes("source.csv", "target.csv", key_columns="id")
+    print(comparison["statistics"]["match_rate_percent"], "% match rate")
+"""
+
+__version__ = "0.1.0"
+
+# Reconciler
+from .reconciler import (
+    compare_hashes,
+    get_orphan_details,
+    get_conflict_details,
+    fuzzy_match_columns,
+    fuzzy_deduplicate,
+    merge_sources,
+    compute_similarity,
+    diff_lists,
+    diff_dicts,
+    explain_diff,
+    find_close_matches,
+    find_similar_strings,
+    transform_column,
+)
+
+# Profiler
+from .profiler import profile_data, detect_schema_drift
+
+# Ingestion
+from .ingestion import load_csv, load_json, extract_pdf_text, parse_table_from_text
+
+# Files
+from .files import find_files, stage_file
+
+__all__ = [
+    "__version__",
+    # Reconciler
+    "compare_hashes",
+    "get_orphan_details",
+    "get_conflict_details",
+    "fuzzy_match_columns",
+    "fuzzy_deduplicate",
+    "merge_sources",
+    "compute_similarity",
+    "diff_lists",
+    "diff_dicts",
+    "explain_diff",
+    "find_close_matches",
+    "find_similar_strings",
+    "transform_column",
+    # Profiler
+    "profile_data",
+    "detect_schema_drift",
+    # Ingestion
+    "load_csv",
+    "load_json",
+    "extract_pdf_text",
+    "parse_table_from_text",
+    # Files
+    "find_files",
+    "stage_file",
+]

databridge_core-0.1.0/src/databridge_core/_io.py

@@ -0,0 +1,8 @@
+"""Shared I/O helpers."""
+
+import pandas as pd
+
+
+def read_csv(file_path: str) -> pd.DataFrame:
+    """Read a CSV file into a DataFrame."""
+    return pd.read_csv(file_path)

databridge_core-0.1.0/src/databridge_core/_types.py

@@ -0,0 +1,125 @@
+"""Shared result types for the databridge-core library.
+
+All library functions return Python objects (dicts, dataclasses, Pydantic models).
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+# -- Profiler types --
+
+class ProfileResult(BaseModel):
+    """Result of profiling a data source."""
+    file: str
+    rows: int
+    columns: int
+    structure_type: str
+    column_types: Dict[str, str]
+    potential_key_columns: List[str]
+    high_cardinality_cols: List[str]
+    low_cardinality_cols: List[str]
+    data_quality: Dict[str, Any]
+    statistics: Dict[str, Any]
+
+
+class DriftResult(BaseModel):
+    """Result of schema drift detection."""
+    source_a: str
+    source_b: str
+    columns_added: List[str]
+    columns_removed: List[str]
+    columns_common: List[str]
+    type_changes: Dict[str, Dict[str, Any]]
+    has_drift: bool
+
+
+# -- Reconciler types --
+
+class CompareHashesResult(BaseModel):
+    """Result of hash-based row comparison."""
+    source_a: Dict[str, Any]
+    source_b: Dict[str, Any]
+    key_columns: List[str]
+    compare_columns: List[str]
+    statistics: Dict[str, Any]
+
+
+class OrphanResult(BaseModel):
+    """Result of orphan record retrieval."""
+    orphan_source: str
+    orphans_in_a: Optional[Dict[str, Any]] = None
+    orphans_in_b: Optional[Dict[str, Any]] = None
+
+
+class ConflictResult(BaseModel):
+    """Result of conflict detail retrieval."""
+    total_conflicts: int
+    showing: int
+    conflicts: List[Dict[str, Any]]
+
+
+class FuzzyMatchResult(BaseModel):
+    """Result of fuzzy column matching."""
+    column_a: str
+    column_b: str
+    threshold: int
+    total_matches: int
+    top_matches: List[Dict[str, Any]]
+
+
+class MergeResult(BaseModel):
+    """Result of merging two sources."""
+    source_a_rows: int
+    source_b_rows: int
+    merged_rows: int
+    merge_type: str
+    columns: List[str]
+    preview: List[Dict[str, Any]]
+
+
+# -- Ingestion types --
+
+class LoadResult(BaseModel):
+    """Result of loading a file."""
+    file: str
+    rows: int
+    columns: List[str]
+    preview: List[Dict[str, Any]]
+    dtypes: Optional[Dict[str, str]] = None
+    null_counts: Optional[Dict[str, int]] = None
+
+
+class PdfExtractResult(BaseModel):
+    """Result of PDF text extraction."""
+    file: str
+    total_pages: int
+    pages_extracted: int
+    content: List[Dict[str, Any]]
+
+
+class OcrResult(BaseModel):
+    """Result of OCR text extraction."""
+    file: str
+    language: str
+    text: str
+    character_count: int
+
+
+class TableParseResult(BaseModel):
+    """Result of parsing tabular data from text."""
+    columns: Optional[List[str]] = None
+    row_count: Optional[int] = None
+    preview: Optional[List[Dict[str, Any]]] = None
+    raw_row: Optional[List[str]] = None
+
+
+class QueryResult(BaseModel):
+    """Result of a database query."""
+    rows_returned: int
+    columns: List[str]
+    dtypes: Optional[Dict[str, str]] = None
+    preview: List[Dict[str, Any]]
+    truncated: bool = False
+    sql: Optional[str] = None