datanyx 0.1.0__py3-none-any.whl

datanyx/__init__.py

@@ -0,0 +1,117 @@
+"""DataNyx: describe data to an AI agent without showing it the data."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from dataclasses import replace
+
+from .keys import Evidence, apply_evidence
+from .privacy import KeyPolicy
+from .renderers import RENDERERS
+from .schema import DataSchema
+from .sources import Source, SourceError, describe_folder, resolve_source
+from .text import DEFAULT_MODEL
+
+__version__ = "0.1.0"
+
+
+def extract(
+    target: str,
+    source: str | None = None,
+    policy: KeyPolicy | None = None,
+    namespace: str | None = None,
+    recursive: bool = False,
+) -> DataSchema:
+    """Build a value-free schema for a file path, "-" for stdin, a database URL or a folder.
+
+    `namespace` picks a database schema; the database's default is used otherwise.
+    For a folder, every readable file in it is described (and in its subfolders,
+    when `recursive` is set); `source` then limits which files are read.
+    """
+    policy = policy or KeyPolicy()
+    if policy.infer_keys and policy.evidence is None:
+        policy = replace(policy, evidence=Evidence())
+    if os.path.isdir(target):
+        if namespace is not None:
+            raise SourceError("a schema can only be selected for SQL databases")
+        schema = describe_folder(target, source, policy, recursive)
+    else:
+        schema = resolve_source(target, source).extract(target, policy, namespace)
+    if policy.infer_keys and policy.evidence is not None:
+        schema = apply_evidence(schema, policy.evidence)
+        if not any(e.foreign_keys or _has_unique(e) for e in schema.entities):
+            schema = replace(schema, notes=(*schema.notes, "no unique columns or joins were inferred; this kind of source may not support it, or no key-looking column qualified"))
+    if policy.scale_hints and not any(entity.scale for entity in schema.entities):
+        schema = replace(schema, notes=(*schema.notes, "no row-count hints: this kind of source can't give one without reading it"))
+    return schema
+
+
+def _has_unique(entity) -> bool:
+    node = getattr(entity.node, "items", entity.node)
+    return any(field.unique for field in getattr(node, "fields", {}).values())
+
+
+def render(schema: DataSchema, output_format: str = "json-schema") -> str:
+    """Render a schema in one of the formats listed in `RENDERERS`."""
+    return RENDERERS[output_format](schema)
+
+
+def schema_of(
+    target: str,
+    output_format: str = "json-schema",
+    *,
+    source: str | None = None,
+    map_paths: Iterable[str] = (),
+    keep_names: Iterable[str] = (),
+    namespace: str | None = None,
+    recursive: bool = False,
+) -> str:
+    """The schema of `target` as text, in one call.
+
+    >>> print(datanyx.schema_of("customers.csv", "text"))
+
+    Raises `SourceError` with a message that never quotes the data. Use `extract` and
+    `render` instead to get the schema as objects, or to see its `notes`.
+    """
+    policy = KeyPolicy(map_paths=frozenset(map_paths), keep_names=frozenset(keep_names))
+    schema = extract(target, source, policy, namespace, recursive)
+    return render(schema, output_format)
+
+
+def anonymize_text(
+    text: str,
+    *,
+    model: str = DEFAULT_MODEL,
+    hide: Iterable[str] = (),
+    keep: Iterable[str] = (),
+    keep_entity: Iterable[str] = (),
+) -> str:
+    """`text` with names and identifiers replaced by placeholders such as <PERSON_1>.
+
+    Needs `datanyx[text]` and a spaCy model. Best effort: review the result.
+    """
+    from .text import Detector, PatternDetector, SpacyDetector, anonymize
+
+    detectors: list[Detector] = [PatternDetector(), SpacyDetector(model)]
+    return anonymize(text, detectors, hide, keep, keep_entity)
+
+
+def synthesize(schema: DataSchema, rows: int = 100, seed: int | None = 0):
+    """Fake data matching `schema`, as a list of `datanyx.synth.Table`. See `datanyx.synth`."""
+    from .synth import synthesize as generate
+
+    return generate(schema, rows, seed)
+
+
+__all__ = [
+    "DataSchema",
+    "KeyPolicy",
+    "Source",
+    "SourceError",
+    "anonymize_text",
+    "extract",
+    "render",
+    "schema_of",
+    "synthesize",
+]

datanyx/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

datanyx/audit.py

@@ -0,0 +1,194 @@
+"""A record of what an agent did through the MCP server: which tool, on what, and how it went.
+
+One JSON object per line. It never holds a value or anything the agent was given back:
+only the tool's name, the paths and settings it was called with, whether it worked, and
+the sizes involved. Arguments that could carry data are reduced to a length or a count
+(the SQL an agent asked to have checked, the terms it asked to hide), and a password in a
+database URL is replaced.
+
+Keep the file where the agent can't change it. `datanyx audit FILE` summarises it.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import uuid
+from collections import Counter
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .fileperms import restrict_to_owner
+from .sources.base import SourceError
+
+_CREDENTIALS = re.compile(r"(?<=://)[^/\s:@]+:[^@\s]+(?=@)")
+_MAX_TEXT = 300
+# How each tool's arguments are recorded. Anything not listed is reduced to its size.
+_KEEP = "keep"
+_COUNT = "count"
+_LENGTH = "length"
+_RULES: dict[str, dict[str, str]] = {
+    "describe_data": {k: _KEEP for k in ("target", "format", "recursive", "map_paths", "database_schema")},
+    "synthetic_sample": {k: _KEEP for k in ("target", "rows", "recursive", "map_paths", "database_schema")},
+    "check_code": {"code": _LENGTH, **{k: _KEEP for k in ("target", "language", "dialect", "table", "database_schema")}},
+    "scan_folder": {"target": _KEEP, "kinds": _KEEP},
+    "schema_diff": {"old": _KEEP, "new": _KEEP},
+    "anonymize_file": {"target": _KEEP, "hide": _COUNT},
+}
+
+
+class AuditLog:
+    """Appends records to a file. With no path it does nothing."""
+
+    def __init__(self, path: str | Path | None) -> None:
+        self.path = Path(path) if path else None
+        self.session = uuid.uuid4().hex[:12]
+        if self.path is not None:
+            self._prepare()
+
+    def _prepare(self) -> None:
+        assert self.path is not None
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            created = not self.path.exists()
+            with open(self.path, "a", encoding="utf-8"):
+                pass
+            if created:
+                restrict_to_owner(str(self.path))
+        except OSError:
+            raise SourceError("cannot write the audit log") from None
+
+    def record(
+        self, tool: str, arguments: Mapping[str, Any], outcome: str, message: str | None, started: float, output_chars: int | None
+    ) -> None:
+        if self.path is None:
+            return
+        entry: dict[str, Any] = {
+            "time": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            "session": self.session,
+            "tool": tool,
+            "arguments": summarize(tool, arguments),
+            "outcome": outcome,
+            "seconds": round(time.monotonic() - started, 3),
+        }
+        if message:
+            entry["message"] = message[:_MAX_TEXT]
+        if output_chars is not None:
+            entry["output_chars"] = output_chars
+        try:
+            with open(self.path, "a", encoding="utf-8") as stream:
+                stream.write(json.dumps(entry, ensure_ascii=True) + chr(10))
+        except OSError:
+            pass  # A full disk shouldn't stop the tool the agent asked for.
+
+
+def summarize(tool: str, arguments: Mapping[str, Any]) -> dict[str, Any]:
+    rules = _RULES.get(tool, {})
+    summary: dict[str, Any] = {}
+    for name, value in arguments.items():
+        mode = rules.get(name)
+        if value is None:
+            continue
+        if mode == _KEEP:
+            summary[name] = _clean(value)
+        elif mode == _COUNT or (mode is None and isinstance(value, (list, tuple, dict))):
+            summary[f"{name}_count"] = len(value)
+        else:
+            summary[f"{name}_length"] = len(value) if hasattr(value, "__len__") else 1
+    return summary
+
+
+def _clean(value: Any) -> Any:
+    if isinstance(value, str):
+        return _CREDENTIALS.sub("<CREDENTIALS>", value)[:_MAX_TEXT]
+    if isinstance(value, (list, tuple)):
+        return [_clean(item) for item in value][:50]
+    return value
+
+
+# --- reading a log back ------------------------------------------------------------------
+
+
+def read_log(path: str | Path, since: str | None = None) -> tuple[list[dict[str, Any]], int]:
+    """The records in a log, and how many lines couldn't be read."""
+    cutoff = _parse_time(since) if since else None
+    records: list[dict[str, Any]] = []
+    unreadable = 0
+    try:
+        lines = Path(path).read_text(encoding="utf-8").splitlines()
+    except OSError:
+        raise SourceError("cannot read the audit log") from None
+    for line in lines:
+        if not line.strip():
+            continue
+        try:
+            entry = json.loads(line)
+            when = _parse_time(entry["time"])
+        except (ValueError, KeyError, TypeError):
+            unreadable += 1
+            continue
+        if cutoff is None or when >= cutoff:
+            records.append(entry)
+    return records, unreadable
+
+
+def _parse_time(text: str) -> datetime:
+    try:
+        parsed = datetime.fromisoformat(text)
+    except ValueError:
+        raise SourceError("times look like 2025-01-31 or 2025-01-31T09:30:00") from None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def summarize_log(records: list[dict[str, Any]], unreadable: int = 0, output_format: str = "text") -> str:
+    tools = Counter(entry.get("tool", "?") for entry in records)
+    sessions = {entry.get("session") for entry in records}
+    targets: Counter[str] = Counter()
+    for entry in records:
+        arguments = entry.get("arguments", {})
+        for key in ("target", "old", "new"):
+            if isinstance(arguments.get(key), str):
+                targets[arguments[key]] += 1
+    failures = Counter(
+        (entry.get("tool", "?"), entry.get("message", "")) for entry in records if entry.get("outcome") != "ok"
+    )
+    times = sorted(entry["time"] for entry in records if "time" in entry)
+
+    if output_format == "json":
+        return json.dumps(
+            {
+                "calls": len(records),
+                "sessions": len(sessions),
+                "first": times[0] if times else None,
+                "last": times[-1] if times else None,
+                "tools": dict(tools),
+                "targets": dict(targets),
+                "failures": [{"tool": tool, "message": message, "count": count} for (tool, message), count in failures.items()],
+                "unreadable_lines": unreadable,
+            },
+            indent=2,
+        )
+    if not records:
+        return "The audit log has no calls" + (" in that period." if unreadable == 0 else f" ({unreadable} unreadable lines).")
+    lines = [
+        f"{len(records):,} call{'s' if len(records) != 1 else ''} in {len(sessions)} session{'s' if len(sessions) != 1 else ''}, "
+        f"{times[0]} to {times[-1]}",
+        "",
+        "By tool: " + " · ".join(f"{tool} {count:,}" for tool, count in tools.most_common()),
+    ]
+    if targets:
+        lines += ["", "Paths asked about:"]
+        lines += [f"  {target}  ×{count}" for target, count in targets.most_common(15)]
+    if failures:
+        lines += ["", f"{sum(failures.values())} call{'s' if sum(failures.values()) != 1 else ''} failed or were refused:"]
+        lines += [f"  {tool}: {message}  ×{count}" for (tool, message), count in failures.most_common(10)]
+    if unreadable:
+        lines += ["", f"note: {unreadable} line{'s' if unreadable != 1 else ''} couldn't be read."]
+    return chr(10).join(lines)
+
+
+__all__ = ["AuditLog", "read_log", "summarize", "summarize_log"]
+

datanyx/check/__init__.py

@@ -0,0 +1,125 @@
+"""Check code against a schema before it runs: do the tables and columns it uses exist?
+
+A schema lists the names that are visible, so this can catch a misspelled column, a
+wrong table, an ambiguous join or a comparison between incompatible types without any
+data being read by the code or by the checker. The code being checked is the agent's own,
+so quoting it back in a message reveals nothing.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Sequence
+
+from ..privacy import KeyPolicy
+from ..sources.base import SourceError
+from .model import Column, Diagnostic, Table, tables_from_schema
+from .python_check import check_python
+from .sql_check import check_sql
+
+LANGUAGES = ("sql", "python")
+_EXTENSIONS = {".sql": "sql", ".py": "python"}
+_DIALECTS = {
+    "postgres": "postgres",
+    "postgresql": "postgres",
+    "mysql": "mysql",
+    "mariadb": "mysql",
+    "mssql": "tsql",
+    "oracle": "oracle",
+    "sqlite": "sqlite",
+}
+
+
+def detect_language(path: str) -> str | None:
+    return _EXTENSIONS.get(os.path.splitext(path)[1].lower())
+
+
+def check_code(
+    code: str, tables: Sequence[Table], language: str, dialect: str | None = None
+) -> list[Diagnostic]:
+    """Findings for `code`, in line order. `language` is `sql` or `python`."""
+    if language == "sql":
+        found = check_sql(code, tables, dialect)
+    elif language == "python":
+        found = check_python(code, tables, dialect)
+    else:
+        raise SourceError(f"language must be one of {list(LANGUAGES)}")
+    return sorted(found, key=lambda item: (item.line, item.column or 0))
+
+
+def load_tables(
+    targets: Sequence[str],
+    table_names: Sequence[str] = (),
+    source: str | None = None,
+    policy: KeyPolicy | None = None,
+    namespace: str | None = None,
+    recursive: bool = False,
+) -> tuple[list[Table], str | None]:
+    """The tables the targets describe, and the SQL dialect they suggest (if they agree).
+
+    A single file has no table name of its own (the schema leaves file names out), so it
+    is called `table_names[i]`, or its file name if that is safe to show, or `data`. Its
+    real file name still works when code refers to it.
+    """
+    from .. import extract  # Imported here: the package imports this module.
+
+    policy = policy or KeyPolicy()
+    tables: list[Table] = []
+    dialects: set[str | None] = set()
+    for position, target in enumerate(targets):
+        schema = extract(target, source, policy, namespace, recursive)
+        dialects.add(_dialect_for(target))
+        is_file = os.path.isfile(target)
+        stem = os.path.splitext(os.path.basename(target))[0] if is_file else ""
+        given = table_names[position] if position < len(table_names) else None
+        name = given or (stem if stem and policy.shows_name(stem) else None)
+        found = tables_from_schema(schema, name)
+        if is_file:
+            base = os.path.basename(target)
+            for table in found:
+                table.aliases = table.aliases | {base, stem}
+        tables.extend(found)
+    return tables, (dialects.pop() if len(dialects) == 1 else None)
+
+
+def _dialect_for(target: str) -> str | None:
+    value = os.environ.get(target[4:], "") if target.startswith("env:") else target
+    match = re.match(r"([A-Za-z][A-Za-z0-9+.-]*)://", value)
+    if match:
+        return _DIALECTS.get(match.group(1).split("+")[0].lower())
+    if os.path.splitext(value)[1].lower() in (".db", ".sqlite", ".sqlite3"):
+        return "sqlite"
+    return None
+
+
+def format_diagnostics(diagnostics: Sequence[Diagnostic], path: str = "<stdin>", output_format: str = "text") -> str:
+    """Findings as text, one per line with an optional hint, or as JSON."""
+    if output_format == "json":
+        return json.dumps([item.as_dict() for item in diagnostics], indent=2)
+    if not diagnostics:
+        return "No problems found."
+    lines = []
+    for item in diagnostics:
+        where = f"{path}:{item.line}" + (f":{item.column}" if item.column else "")
+        lines.append(f"{where}: {item.severity}: {item.message}")
+        if item.hint:
+            lines.append(f"    hint: {item.hint}")
+    errors = sum(item.severity == "error" for item in diagnostics)
+    warnings = len(diagnostics) - errors
+    lines.append(f"{errors} error{'s' if errors != 1 else ''}, {warnings} warning{'s' if warnings != 1 else ''}")
+    return chr(10).join(lines)
+
+
+__all__ = [
+    "LANGUAGES",
+    "Column",
+    "Diagnostic",
+    "Table",
+    "check_code",
+    "detect_language",
+    "format_diagnostics",
+    "load_tables",
+    "tables_from_schema",
+]

datanyx/check/model.py

@@ -0,0 +1,179 @@
+"""What the checkers know about the data: tables, columns and their kinds.
+
+Everything here comes from a `DataSchema`, so it holds only names the schema shows.
+That matters for what a check can reveal: a name that is hidden is simply absent, so
+asking about it gives the same answer as asking about a name that doesn't exist. A
+checker can't be used to test guesses about hidden names.
+"""
+
+from __future__ import annotations
+
+import difflib
+import os
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+from ..schema import (
+    Array,
+    Boolean,
+    DataSchema,
+    Entity,
+    Integer,
+    Node,
+    Null,
+    Number,
+    Object,
+    String,
+    Union,
+    Unknown,
+)
+
+NUMERIC = frozenset({"integer", "number"})
+TEXT = frozenset({"string", "uuid"})
+TEMPORAL = frozenset({"date", "datetime", "time"})
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    """One finding. `error` means the code can't work; `warning` means it probably doesn't."""
+
+    severity: str
+    message: str
+    line: int = 1
+    column: int | None = None
+    hint: str | None = None
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "severity": self.severity,
+            "message": self.message,
+            "line": self.line,
+            "column": self.column,
+            "hint": self.hint,
+        }
+
+
+@dataclass(frozen=True)
+class Column:
+    """`kind` is one of integer, number, boolean, string, uuid, date, datetime, time, binary,
+    array, object, mixed or unknown."""
+
+    kind: str = "unknown"
+    nullable: bool = True
+
+    @property
+    def known(self) -> bool:
+        return self.kind not in ("unknown", "mixed")
+
+
+@dataclass
+class Table:
+    """A table, or anything table-shaped: a CSV file, a sheet, a list of JSON records."""
+
+    name: str
+    columns: dict[str, Column]
+    #: True when some column names are hidden, so a name missing here may still exist.
+    open: bool = False
+    #: Other names the table answers to, such as a file's name with its extension.
+    aliases: frozenset[str] = frozenset()
+    #: Where the data came from (`csv`, `json`, `sql`...), which says how a reader will type it.
+    source_kind: str = ""
+    _lower: dict[str, str] = field(default_factory=dict, repr=False)
+
+    def __post_init__(self) -> None:
+        self._lower = {name.lower(): name for name in self.columns}
+
+    def column(self, name: str, case_sensitive: bool = False) -> Column | None:
+        if name in self.columns:
+            return self.columns[name]
+        if not case_sensitive and name.lower() in self._lower:
+            return self.columns[self._lower[name.lower()]]
+        return None
+
+    def answers_to(self, name: str) -> bool:
+        wanted = name.lower()
+        return wanted == self.name.lower() or wanted in {alias.lower() for alias in self.aliases}
+
+
+def tables_from_schema(schema: DataSchema, single_name: str | None = None) -> list[Table]:
+    """One `Table` per entity. An unnamed entity is called `single_name`, or `data`."""
+    tables = []
+    for entity in schema.entities:
+        node = entity.node.items if isinstance(entity.node, Array) else entity.node
+        if not isinstance(node, Object):
+            continue  # A list of plain values has no columns to check.
+        name = entity.name or single_name or "data"
+        tables.append(
+            Table(
+                name,
+                {column: _column(field.node) for column, field in node.fields.items()},
+                node.extra is not None,
+                frozenset(_aliases(name)),
+                _source_kind(schema.source_kind, name),
+            )
+        )
+    return tables
+
+
+_EXTENSION_KINDS = {
+    ".csv": "csv", ".tsv": "tsv", ".json": "json", ".jsonl": "jsonl", ".ndjson": "jsonl", ".yaml": "yaml", ".yml": "yaml", ".xml": "xml",
+}
+
+
+def _source_kind(kind: str, name: str) -> str:
+    """For a folder, the kind comes from each file's extension."""
+    if kind != "directory":
+        return kind
+    return _EXTENSION_KINDS.get(os.path.splitext(name)[1].lower(), "")
+
+
+def _aliases(name: str) -> set[str]:
+    """`reports/orders.csv` also answers to `orders.csv` and `orders`; `a.db#items` to `items`."""
+    last = re.split(r"[/#\\]", name)[-1]
+    return {name, last, last.split(".")[0] if "." in last else last}
+
+
+def _column(node: Node) -> Column:
+    options = node.options if isinstance(node, Union) else (node,)
+    nullable = any(isinstance(option, (Null, Unknown)) for option in options)
+    kinds = {_kind(option) for option in options if not isinstance(option, (Null, Unknown))}
+    if not kinds:
+        return Column("unknown", nullable)
+    if len(kinds) == 1:
+        return Column(next(iter(kinds)), nullable)
+    return Column("number" if kinds <= NUMERIC else "mixed", nullable)
+
+
+def _kind(node: Node) -> str:
+    if isinstance(node, Integer):
+        return "integer"
+    if isinstance(node, Number):
+        return "number"
+    if isinstance(node, Boolean):
+        return "boolean"
+    if isinstance(node, String):
+        return {"date": "date", "date-time": "datetime", "time": "time", "uuid": "uuid", "binary": "binary"}.get(
+            node.format or "", "string"
+        )
+    if isinstance(node, Array):
+        return "array"
+    if isinstance(node, Object):
+        return "object"
+    return "unknown"
+
+
+def suggest(name: str, candidates: Iterable[str], case_sensitive: bool = False) -> str | None:
+    """A hint for a name that wasn't found: the same name in other case, or a close one."""
+    options = list(candidates)
+    for option in options:
+        if option.lower() == name.lower() and option != name:
+            return f"did you mean {option!r}? Names are case-sensitive here" if case_sensitive else f"did you mean {option!r}?"
+    close = difflib.get_close_matches(name, options, n=3, cutoff=0.6)
+    if not close:
+        return None
+    return "did you mean " + " or ".join(repr(match) for match in close) + "?"
+
+
+def describe(entity: Entity) -> str:
+    return entity.name or "data"