binggio 0.1.0__py3-none-any.whl

binggio/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 Ashish Kumar
+"""Binggio: governed answers from your data.
+
+Binggio answers questions only from meanings a person has approved (column roles,
+relationships, business terms, metrics), and refuses with the reason when it cannot.
+"""
+__version__ = '0.1.0'
+
+from binggio.workspace import Workspace, WorkspaceError  # noqa: E402,F401

binggio/__main__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 Ashish Kumar
+import sys
+from binggio.cli import main
+sys.exit(main())

binggio/catalog/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 Ashish Kumar
+"""Data catalog: registered sources and their tables."""

binggio/catalog/catalog.py

@@ -0,0 +1,206 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 Ashish Kumar
+"""Persistent Phase4 local catalog. No auto-discovery or approval claims.
+
+Metadata scans are bounded, read-only and stored in a separate SQLite catalog.
+This module never imports test-only ground truth.
+"""
+from __future__ import annotations
+import json
+import sqlite3
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from .connectors import (ConnectionErrorSafe, list_csv_tables, list_sqlite_tables,
+                         read_csv_metadata, read_sqlite_metadata, resolve_source)
+
+_LOCK = threading.RLock()
+DEFAULTS = []  # the package registers no seed sources
+_POC_DEFAULTS = [
+    ("csv_clean", "CSV — clean reference", "csv_folder", "phase4_workbench/csv_clean"),
+    ("sqlite_imperfect", "SQLite — imperfect reference", "sqlite", "phase4_workbench/sqlite_imperfect/enterprise_demo.db"),
+    ("csv_messy", "CSV — messy reference", "csv_folder", "phase4_workbench/csv_messy"),
+]
+
+
+def _utc() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _db(project: Path) -> sqlite3.Connection:
+    target = Path(project).resolve() / "data" / "phase4_catalog" / "catalog.sqlite3"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(target, timeout=10)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA foreign_keys = ON")
+    conn.execute("PRAGMA busy_timeout = 5000")
+    conn.execute("""CREATE TABLE IF NOT EXISTS connections (
+        id TEXT PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL,
+        source_path TEXT NOT NULL, created_at TEXT NOT NULL, last_scan TEXT,
+        scan_status TEXT NOT NULL DEFAULT 'pending', last_error TEXT,
+        UNIQUE(kind, source_path))""")
+    conn.execute("""CREATE TABLE IF NOT EXISTS catalog_tables (
+        connection_id TEXT NOT NULL, schema_name TEXT NOT NULL,
+        table_name TEXT NOT NULL, row_count INTEGER, column_count INTEGER,
+        scan_status TEXT NOT NULL, warnings_json TEXT NOT NULL,
+        columns_json TEXT NOT NULL, scanned_at TEXT NOT NULL,
+        PRIMARY KEY(connection_id, schema_name, table_name),
+        FOREIGN KEY(connection_id) REFERENCES connections(id) ON DELETE CASCADE)""")
+    conn.commit()
+    return conn
+
+
+def init_catalog(project: Path) -> None:
+    with _LOCK, _db(project) as db:
+        for cid, name, kind, rel in DEFAULTS:
+            try:resolve_source(project, rel, kind)
+            except ConnectionErrorSafe:continue
+            db.execute("""INSERT OR IGNORE INTO connections
+                (id,name,kind,source_path,created_at,scan_status)
+                VALUES (?,?,?,?,?,'pending')""", (cid,name,kind,rel,_utc()))
+        db.commit()
+        for cid, *_ in DEFAULTS:
+            row = db.execute("SELECT id, last_scan FROM connections WHERE id=?", (cid,)).fetchone()
+            if row and row["last_scan"] is None:
+                refresh_connection(project, cid)
+
+
+def list_connections(project: Path) -> list[dict]:
+    with _LOCK, _db(project) as db:
+        return [dict(r) for r in db.execute("""SELECT c.*,
+            (SELECT COUNT(*) FROM catalog_tables t WHERE t.connection_id=c.id) AS table_count
+            FROM connections c ORDER BY c.created_at,c.name""").fetchall()]
+
+
+def add_connection(project: Path, *, name: str, kind: str, source_path: str) -> dict:
+    name = str(name or "").strip()
+    if not (1 <= len(name) <= 80):raise ConnectionErrorSafe("Connection name must contain 1–80 characters")
+    resolve_source(project, source_path, kind)
+    cid = "conn_" + uuid.uuid4().hex[:14]
+    with _LOCK, _db(project) as db:
+        try:
+            db.execute("INSERT INTO connections (id,name,kind,source_path,created_at) VALUES (?,?,?,?,?)",
+                       (cid,name,kind,source_path,_utc()))
+            db.commit()
+        except sqlite3.IntegrityError as exc:
+            raise ConnectionErrorSafe("This source is already registered") from exc
+    try:
+        refresh_connection(project, cid)
+    except Exception:
+        # Keep connection with failed scan status for diagnostic visibility.
+        pass
+    return next(c for c in list_connections(project) if c["id"] == cid)
+
+
+def remove_connection(project: Path, cid: str) -> None:
+    if cid in {c[0] for c in DEFAULTS}:
+        raise ConnectionErrorSafe("Seed connections cannot be deleted; add another source instead")
+    with _LOCK, _db(project) as db:
+        cur = db.execute("DELETE FROM connections WHERE id=?", (cid,))
+        db.commit()
+        if not cur.rowcount:raise ConnectionErrorSafe("Connection not found")
+
+
+def refresh_connection(project: Path, cid: str) -> dict:
+    with _LOCK, _db(project) as db:
+        row = db.execute("SELECT * FROM connections WHERE id=?", (cid,)).fetchone()
+        if row is None:raise ConnectionErrorSafe("Connection not found")
+        try:
+            source = resolve_source(project, row["source_path"], row["kind"])
+            names = list_csv_tables(source) if row["kind"] == "csv_folder" else list_sqlite_tables(source)
+            if not names:raise ConnectionErrorSafe("No supported tables found")
+            staged = []
+            failures = 0
+            for name in names:
+                try:
+                    if row["kind"] == "csv_folder":
+                        p = source / (name + ".csv")
+                        if not p.is_file() or not p.resolve().is_relative_to(source):
+                            raise ConnectionErrorSafe("Source CSV escaped folder or was removed")
+                        meta = read_csv_metadata(p, preview=0)
+                    else:meta = read_sqlite_metadata(source, name, preview=0)
+                except (OSError, ValueError, sqlite3.Error) as exc:
+                    meta = {"columns": [], "row_count": None, "warnings": [str(exc)[:180]],
+                            "scan_status": "error"}
+                    failures += 1
+                staged.append((cid, "files" if row["kind"] == "csv_folder" else "main", name,
+                               meta.get("row_count"), len(meta["columns"]),meta["scan_status"],
+                               json.dumps(meta["warnings"]), json.dumps(meta["columns"]),_utc()))
+            # Swap metadata atomically; failed connection scans don't discard old state.
+            db.execute("DELETE FROM catalog_tables WHERE connection_id=?", (cid,))
+            db.executemany("""INSERT INTO catalog_tables
+                (connection_id,schema_name,table_name,row_count,column_count,scan_status,
+                 warnings_json,columns_json,scanned_at) VALUES (?,?,?,?,?,?,?,?,?)""", staged)
+            db.execute("UPDATE connections SET last_scan=?,scan_status=?,last_error=NULL WHERE id=?",
+                       (_utc(),"partial" if failures else "scanned",cid))
+            db.commit()
+        except Exception as exc:
+            db.execute("UPDATE connections SET scan_status='error',last_error=? WHERE id=?",(str(exc)[:220],cid))
+            db.commit()
+            raise
+    return next(c for c in list_connections(project) if c["id"] == cid)
+
+
+
+def list_schemas(project: Path, connection_id: str | None = None) -> list[dict]:
+    with _LOCK, _db(project) as db:
+        sql = """SELECT connection_id,schema_name,COUNT(*) AS table_count
+                 FROM catalog_tables WHERE 1=1"""
+        args = []
+        if connection_id:
+            sql += " AND connection_id=?";args.append(connection_id)
+        sql += " GROUP BY connection_id,schema_name ORDER BY connection_id,schema_name"
+        return [dict(r) for r in db.execute(sql,args).fetchall()]
+
+def list_tables(project: Path, *, connection_id: str | None = None, search: str = "", limit: int = 250) -> list[dict]:
+    limit = min(max(int(limit),1), 5000)
+    with _LOCK, _db(project) as db:
+        sql = """SELECT t.connection_id,t.schema_name,t.table_name,t.row_count,t.column_count,
+                 t.scan_status,t.warnings_json,t.scanned_at,c.name AS connection_name,c.kind
+                 FROM catalog_tables t JOIN connections c ON c.id=t.connection_id WHERE 1=1"""
+        args: list[Any] = []
+        if connection_id:
+            sql += " AND t.connection_id=?";args.append(connection_id)
+        if search:
+            sql += " AND LOWER(t.table_name) LIKE ?";args.append("%"+search.lower().replace("%","\\%").replace("_","\\_")+"%")
+            sql += " ESCAPE '\\'"
+        sql += " ORDER BY t.connection_id,t.table_name LIMIT ?";args.append(limit)
+        result=[]
+        for row in db.execute(sql,args).fetchall():
+            d=dict(row);d["warnings"]=json.loads(d.pop("warnings_json"));result.append(d)
+        return result
+
+
+def table_detail(project: Path, cid: str, name: str, *, preview: int = 10) -> dict:
+    preview = min(max(preview,0),25)
+    with _LOCK, _db(project) as db:
+        c=db.execute("SELECT * FROM connections WHERE id=?",(cid,)).fetchone()
+        if not c:raise ConnectionErrorSafe("Connection not found")
+        t=db.execute("SELECT * FROM catalog_tables WHERE connection_id=? AND table_name=?",(cid,name)).fetchone()
+        if not t:raise ConnectionErrorSafe("Table is not in the catalog; refresh the connection")
+        source=resolve_source(project,c["source_path"],c["kind"])
+        if c["kind"] == "csv_folder":
+            p=(source/(name+".csv")).resolve()
+            if not p.is_relative_to(source):raise ConnectionErrorSafe("CSV source escaped folder")
+            meta=read_csv_metadata(p,preview=preview)
+        else:meta=read_sqlite_metadata(source,name,preview=preview)
+        return {"connection_id":cid,"connection_name":c["name"],"schema":t["schema_name"],
+                "table":name,"source_kind":c["kind"],**meta,
+                "note":"Basic technical metadata only. Semantic roles and relationships are not yet verified."}
+
+
+def overview(project: Path) -> dict:
+    with _LOCK, _db(project) as db:
+        connections=db.execute("SELECT COUNT(*) FROM connections").fetchone()[0]
+        tables=db.execute("SELECT COUNT(*) FROM catalog_tables").fetchone()[0]
+        columns=db.execute("SELECT COALESCE(SUM(column_count),0) FROM catalog_tables").fetchone()[0]
+        warnings=db.execute("SELECT warnings_json FROM catalog_tables").fetchall()
+        warning_tables=sum(bool(json.loads(r[0])) for r in warnings)
+        errors=db.execute("SELECT COUNT(*) FROM catalog_tables WHERE scan_status!='scanned'").fetchone()[0]
+        return {"connections":connections,"tables":tables,"columns":columns,
+                "tables_with_warnings":warning_tables,"tables_with_scan_errors":errors,
+                "phase":"4","data_label":"Synthetic local demonstration",
+                "relationship_discovery":"not_implemented","human_review":"not_implemented",
+                "session_memory":"planned_phase_8B"}

binggio/catalog/connectors.py

@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 Ashish Kumar
+"""Phase 4 local CSV and SQLite read-only metadata connectors.
+
+No arbitrary SQL endpoint, no external credentials, no network connections.
+Path sandbox is <project>/data only. This is not production connector infrastructure.
+"""
+from __future__ import annotations
+import csv
+import io
+import os
+import re
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+MAX_CSV_BYTES = 2 * 1024 * 1024 * 1024  # 2 GB per CSV table; files are streamed, never loaded whole
+MAX_PREVIEW = 25
+MAX_COLUMNS = 200
+MAX_TABLES = 2000
+
+
+class ConnectionErrorSafe(ValueError):
+    pass
+
+
+def resolve_source(project: Path, relative_path: str, kind: str) -> Path:
+    if kind not in ("csv_folder", "sqlite"):
+        raise ConnectionErrorSafe("Supported connection types are CSV folder and SQLite database")
+    rel = str(relative_path or "").strip().replace("\\", "/")
+    if not rel or os.path.isabs(rel) or re.match(r"^[A-Za-z]:", rel):
+        raise ConnectionErrorSafe("Use a relative path under the project's data/ folder")
+    root = (Path(project) / "data").resolve()
+    candidate = (root / rel).resolve()
+    if candidate == root or not candidate.is_relative_to(root):
+        raise ConnectionErrorSafe("Connection path must be inside a subfolder of data/")
+    if kind == "csv_folder" and not candidate.is_dir():
+        raise ConnectionErrorSafe("CSV folder does not exist")
+    if kind == "sqlite" and (not candidate.is_file() or candidate.suffix.lower() not in (".db", ".sqlite", ".sqlite3")):
+        raise ConnectionErrorSafe("SQLite database does not exist or has an unsupported extension")
+    return candidate
+
+
+def _unwrapped_row(row: list[str], delim: str) -> list[str]:
+    # Some legacy exports wrap an entire comma-separated line in one quoted field.
+    if len(row) == 1 and delim in row[0]:
+        try:
+            parsed = next(csv.reader([row[0]], delimiter=delim))
+            if len(parsed) > 1:return parsed
+        except (csv.Error, StopIteration):pass
+    return row
+
+
+def read_csv_metadata(path: Path, *, preview: int = MAX_PREVIEW) -> dict[str, Any]:
+    warnings: list[str] = []
+    if path.stat().st_size > MAX_CSV_BYTES:
+        return {"columns": [], "row_count": None, "preview": [],
+                "warnings": ["File exceeds the 2 GB per-table limit; not scanned"],
+                "scan_status": "limited"}
+    try:
+        handle = path.open("r", encoding="utf-8-sig", errors="replace", newline="")
+        first = handle.read(12000)
+    except OSError as exc:
+        raise ConnectionErrorSafe("Cannot read CSV file") from exc
+    if not first.strip():
+        handle.close()
+        return {"columns": [], "row_count": 0, "preview": [],
+                "warnings": ["Empty CSV"], "scan_status": "limited"}
+    handle.seek(0)
+    try:
+        dialect = csv.Sniffer().sniff(first, delimiters=",;\t|")
+        delimiter = dialect.delimiter
+    except csv.Error:
+        delimiter = ";" if first.splitlines()[0].count(";") > first.splitlines()[0].count(",") else ","
+        warnings.append("Delimiter inferred heuristically")
+    try:
+        reader = csv.reader(handle, delimiter=delimiter)
+        header = _unwrapped_row(next(reader), delimiter)
+    except (csv.Error, StopIteration) as exc:
+        raise ConnectionErrorSafe("Cannot parse CSV header") from exc
+    header = [str(x).strip().lstrip("\ufeff") for x in header]
+    if len(header) > MAX_COLUMNS:
+        return {"columns": [], "row_count": None, "preview": [],
+                "warnings": ["More than 200 columns; scan stopped"], "scan_status": "limited"}
+    if len(set(x.casefold() for x in header)) != len(header):
+        warnings.append("Duplicate column names detected")
+    if any(not h for h in header):warnings.append("Blank column names detected")
+    if len(header) < 2:warnings.append("Single-column CSV; verify delimiter and export formatting")
+    rows = []
+    row_count = 0
+    replaced = False
+    malformed = 0
+    padded_cells = 0
+    blanks = [0] * len(header)
+    sample_values: list[list[str]] = [[] for _ in header]
+    try:
+        for raw in reader:
+            if not raw or not any(x.strip() for x in raw):continue
+            row = _unwrapped_row(raw, delimiter)
+            row_count += 1
+            if len(row) != len(header):malformed += 1
+            padded_cells += sum(str(v) != str(v).strip() for v in row)
+            if not replaced and any("\ufffd" in str(v) for v in row):
+                replaced = True; warnings.append("Some invalid UTF-8 characters were replaced during scan")
+            values = [str(row[i]).strip() if i < len(row) else "" for i in range(len(header))]
+            for i, v in enumerate(values):
+                if not v or v.upper() in ("NULL", "N/A", "NA"):
+                    blanks[i] += 1
+                if len(sample_values[i]) < 100:sample_values[i].append(v)
+            if len(rows) < preview:rows.append(dict(zip(header, values)))
+    except csv.Error:
+        warnings.append("CSV contains malformed quoting; partial scan only")
+    finally:
+        handle.close()
+    if malformed:warnings.append(f"{malformed} rows differ from header width")
+    if padded_cells:warnings.append(f"{padded_cells} values have leading/trailing whitespace")
+    if any(blanks):warnings.append("Missing or null-like values observed; detailed profiling is Phase 5")
+    def inferred(values: list[str]) -> str:
+        present = [v for v in values if v and v.upper() not in ("NULL", "NA", "N/A")]
+        if not present:return "unknown"
+        if all(re.fullmatch(r"[+-]?\d+", v) for v in present):return "integer-like"
+        if all(re.fullmatch(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", v) for v in present):return "decimal-like"
+        if all(re.fullmatch(r"\d{4}[-/]\d{2}[-/]\d{2}", v) for v in present):return "date-like"
+        return "text/mixed"
+    columns = [{"name": h, "declared_type": None, "observed_type": inferred(sample_values[i]),
+                "missing_count": blanks[i], "missing_pct": round(100 * blanks[i] / row_count, 1) if row_count else 0}
+               for i, h in enumerate(header)]
+    return {"columns": columns, "row_count": row_count, "preview": rows,
+            "warnings": warnings, "scan_status": "scanned", "delimiter": "TAB" if delimiter == "\t" else delimiter}
+
+
+def list_csv_tables(folder: Path) -> list[str]:
+    return [p.stem for p in sorted(folder.iterdir(), key=lambda p:p.name.lower())
+            if p.is_file() and p.suffix.lower() == ".csv"][:MAX_TABLES]
+
+
+def _readonly(db: Path) -> sqlite3.Connection:
+    # mode=ro ensures metadata/preview calls cannot mutate customer-provided DBs.
+    conn = sqlite3.connect(db.as_uri() + "?mode=ro", uri=True, timeout=5)
+    conn.execute("PRAGMA query_only = ON")
+    conn.execute("PRAGMA busy_timeout = 5000")
+    return conn
+
+
+def _quote_identifier(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def list_sqlite_tables(db: Path) -> list[str]:
+    with _readonly(db) as conn:
+        return [r[0] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name LIMIT ?",
+            (MAX_TABLES,)).fetchall()]
+
+
+def read_sqlite_metadata(db: Path, table: str, *, preview: int = MAX_PREVIEW) -> dict[str, Any]:
+    with _readonly(db) as conn:
+        allowed = set(list_sqlite_tables(db))
+        if table not in allowed:raise ConnectionErrorSafe("Table not found in this SQLite connection")
+        quoted = _quote_identifier(table)
+        rawcols = conn.execute(f"PRAGMA table_info({quoted})").fetchall()
+        cols = [r[1] for r in rawcols]
+        if len(cols) > MAX_COLUMNS:
+            return {"columns": [], "row_count": None, "preview": [],
+                    "warnings": ["More than 200 columns; scan stopped"], "scan_status": "limited"}
+        count = conn.execute(f"SELECT COUNT(*) FROM {quoted}").fetchone()[0]
+        data = conn.execute(f"SELECT * FROM {quoted} LIMIT ?", (min(MAX_PREVIEW, max(0, preview)),)).fetchall()
+        columns = [{"name": r[1], "declared_type": r[2] or None, "observed_type": "not profiled",
+                    "missing_count": None, "missing_pct": None, "primary_key": bool(r[5])} for r in rawcols]
+        return {"columns": columns, "row_count": count,
+                "preview": [{cols[i]:str(v) if v is not None else None for i,v in enumerate(row)} for row in data],
+                "warnings": [], "scan_status": "scanned"}