datanyx 0.1.0__tar.gz

datanyx-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ludenberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datanyx-0.1.0/PKG-INFO

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: datanyx
+Version: 0.1.0
+Summary: Turn sensitive data into a value-free schema that AI agents can work from.
+License-Expression: MIT
+Keywords: ai-agents,privacy,schema,anonymization,data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy>=2
+Requires-Dist: tomli>=2; python_version < "3.11"
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.1; extra == "postgres"
+Provides-Extra: mysql
+Requires-Dist: pymysql>=1.1; extra == "mysql"
+Requires-Dist: cryptography; extra == "mysql"
+Provides-Extra: mssql
+Requires-Dist: pymssql>=2.3; extra == "mssql"
+Provides-Extra: oracle
+Requires-Dist: oracledb>=2; extra == "oracle"
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=14; extra == "parquet"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1; extra == "excel"
+Provides-Extra: yaml
+Requires-Dist: pyyaml>=6; extra == "yaml"
+Provides-Extra: streaming
+Requires-Dist: ijson>=3.2; extra == "streaming"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.2; extra == "mcp"
+Provides-Extra: avro
+Requires-Dist: fastavro>=1.9; extra == "avro"
+Provides-Extra: orc
+Requires-Dist: pyarrow>=14; extra == "orc"
+Provides-Extra: duckdb
+Requires-Dist: duckdb>=1.0; extra == "duckdb"
+Provides-Extra: stats
+Requires-Dist: pyreadstat>=1.2; (sys_platform != "win32" or python_version >= "3.11") and extra == "stats"
+Provides-Extra: mongodb
+Requires-Dist: pymongo>=4.6; extra == "mongodb"
+Provides-Extra: all
+Requires-Dist: datanyx[avro,check,duckdb,excel,mcp,mongodb,mssql,mysql,oracle,orc,parquet,pdf,postgres,presidio,stats,streaming,yaml]; extra == "all"
+Provides-Extra: text
+Requires-Dist: spacy>=3.7; extra == "text"
+Provides-Extra: check
+Requires-Dist: sqlglot>=25; extra == "check"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=4; extra == "pdf"
+Provides-Extra: presidio
+Requires-Dist: presidio-analyzer>=2.2; extra == "presidio"
+Dynamic: license-file
+
+<div align="center">
+
+# DataNyx
+
+**Let an AI agent work with your data without ever seeing it.**
+
+DataNyx turns files and databases into value-free schemas an agent can write code against,
+and replaces names and identifiers in free text with placeholders.
+
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776ab.svg)](pyproject.toml)
+[![Code style: ruff](https://img.shields.io/badge/lint-ruff-261230.svg)](https://docs.astral.sh/ruff/)
+[![Types: mypy](https://img.shields.io/badge/types-mypy-2a6db2.svg)](https://mypy-lang.org/)
+
+[Getting started](docs/getting-started.md) ·
+[Documentation](docs/index.md) ·
+[Command reference](docs/cli-reference.md) ·
+[Security model](docs/security.md) ·
+[Changelog](CHANGELOG.md)
+
+</div>
+
+---
+
+You want an agent to write a report, a query or a data-cleaning script, but the data is customers,
+patients or payroll. Pasting rows into a chat is a leak, and describing the columns by hand is
+tedious. DataNyx sits in between:
+
+|  | What the agent gets | What it never gets |
+|---|---|---|
+| **Structured data**<br>`datanyx schema` | Field names, types, formats, nesting, primary and foreign keys | Values, examples, enum labels, counts or statistics |
+| **Free text**<br>`datanyx anonymize` | The text with `<PERSON_1>`, `<EMAIL_1>` and friends in place of identifiers | The identifiers themselves (best effort: [review the result](docs/anonymization.md#how-reliable-text-anonymization-is)) |
+
+## A first look
+
+```console
+$ datanyx schema customers.csv --format text
+root: array<{
+  id: integer
+  email: string<email>
+  signed_up: string<date>
+  plan: string
+  lifetime_value: number
+  notes: string | null
+}>
+
+$ datanyx anonymize call-notes.txt
+<PERSON_1> (<EMAIL_1>) called on <DATE_1> about invoice
+<ID_1>. Ms. <PERSON_1> lives in <LOCATION_1>. Her manager <PERSON_2> agreed.
+```
+
+The agent sees the shape of the table and the story in the text, and none of the values. It can also
+try its code on [fake rows](docs/synthetic-data.md) generated from the schema.
+
+## Install
+
+```sh
+git clone <this repository> datanyx && cd datanyx
+uv sync --all-extras                 # or: pip install -e ".[all]"
+datanyx doctor                       # shows which optional packages and models are installed
+```
+
+Python 3.10 or newer. [Supported sources](docs/sources.md) lists the extra for each format.
+
+## What's in the documentation
+
+| | |
+|---|---|
+| [Getting started](docs/getting-started.md) | Install, a first schema, a first anonymized text |
+| [Schemas](docs/schemas.md), [synthetic data](docs/synthetic-data.md) | The core: what the agent sees instead of the data |
+| [Debugging](docs/debugging.md), [checking code](docs/checking-code.md) | Fixing code without seeing the data |
+| [Scanning](docs/scanning.md), [anonymization](docs/anonymization.md), [pseudonymization](docs/pseudonymization.md), [detection quality](docs/detection.md) | Finding and hiding personal data |
+| [Using it with an agent](docs/agents.md) | `datanyx protect`, the guard hook and the MCP server |
+| [Installing and deploying](docs/deployment.md) | PyPI, Docker, a pre-commit hook, a GitHub Action, a shared MCP server |
+| [Team settings](docs/settings.md), [schema diff](docs/schema-diff.md) | Shared policy, and change detection for CI |
+| [Security model](docs/security.md) | What is guaranteed, what isn't, and the known limits |
+| [Command reference](docs/cli-reference.md), [Python API](docs/python-api.md) | Every option, and use from code |
+| [Contributing](docs/contributing.md) | Architecture, adding a source, tests |
+
+To read it as a site: `uv sync --group docs` then `uv run mkdocs serve`.
+
+## License
+
+[MIT](LICENSE) © 2026 ludenberg

datanyx-0.1.0/README.md

@@ -0,0 +1,84 @@
+<div align="center">
+
+# DataNyx
+
+**Let an AI agent work with your data without ever seeing it.**
+
+DataNyx turns files and databases into value-free schemas an agent can write code against,
+and replaces names and identifiers in free text with placeholders.
+
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776ab.svg)](pyproject.toml)
+[![Code style: ruff](https://img.shields.io/badge/lint-ruff-261230.svg)](https://docs.astral.sh/ruff/)
+[![Types: mypy](https://img.shields.io/badge/types-mypy-2a6db2.svg)](https://mypy-lang.org/)
+
+[Getting started](docs/getting-started.md) ·
+[Documentation](docs/index.md) ·
+[Command reference](docs/cli-reference.md) ·
+[Security model](docs/security.md) ·
+[Changelog](CHANGELOG.md)
+
+</div>
+
+---
+
+You want an agent to write a report, a query or a data-cleaning script, but the data is customers,
+patients or payroll. Pasting rows into a chat is a leak, and describing the columns by hand is
+tedious. DataNyx sits in between:
+
+|  | What the agent gets | What it never gets |
+|---|---|---|
+| **Structured data**<br>`datanyx schema` | Field names, types, formats, nesting, primary and foreign keys | Values, examples, enum labels, counts or statistics |
+| **Free text**<br>`datanyx anonymize` | The text with `<PERSON_1>`, `<EMAIL_1>` and friends in place of identifiers | The identifiers themselves (best effort: [review the result](docs/anonymization.md#how-reliable-text-anonymization-is)) |
+
+## A first look
+
+```console
+$ datanyx schema customers.csv --format text
+root: array<{
+  id: integer
+  email: string<email>
+  signed_up: string<date>
+  plan: string
+  lifetime_value: number
+  notes: string | null
+}>
+
+$ datanyx anonymize call-notes.txt
+<PERSON_1> (<EMAIL_1>) called on <DATE_1> about invoice
+<ID_1>. Ms. <PERSON_1> lives in <LOCATION_1>. Her manager <PERSON_2> agreed.
+```
+
+The agent sees the shape of the table and the story in the text, and none of the values. It can also
+try its code on [fake rows](docs/synthetic-data.md) generated from the schema.
+
+## Install
+
+```sh
+git clone <this repository> datanyx && cd datanyx
+uv sync --all-extras                 # or: pip install -e ".[all]"
+datanyx doctor                       # shows which optional packages and models are installed
+```
+
+Python 3.10 or newer. [Supported sources](docs/sources.md) lists the extra for each format.
+
+## What's in the documentation
+
+| | |
+|---|---|
+| [Getting started](docs/getting-started.md) | Install, a first schema, a first anonymized text |
+| [Schemas](docs/schemas.md), [synthetic data](docs/synthetic-data.md) | The core: what the agent sees instead of the data |
+| [Debugging](docs/debugging.md), [checking code](docs/checking-code.md) | Fixing code without seeing the data |
+| [Scanning](docs/scanning.md), [anonymization](docs/anonymization.md), [pseudonymization](docs/pseudonymization.md), [detection quality](docs/detection.md) | Finding and hiding personal data |
+| [Using it with an agent](docs/agents.md) | `datanyx protect`, the guard hook and the MCP server |
+| [Installing and deploying](docs/deployment.md) | PyPI, Docker, a pre-commit hook, a GitHub Action, a shared MCP server |
+| [Team settings](docs/settings.md), [schema diff](docs/schema-diff.md) | Shared policy, and change detection for CI |
+| [Security model](docs/security.md) | What is guaranteed, what isn't, and the known limits |
+| [Command reference](docs/cli-reference.md), [Python API](docs/python-api.md) | Every option, and use from code |
+| [Contributing](docs/contributing.md) | Architecture, adding a source, tests |
+
+To read it as a site: `uv sync --group docs` then `uv run mkdocs serve`.
+
+## License
+
+[MIT](LICENSE) © 2026 ludenberg

datanyx-0.1.0/pyproject.toml

@@ -0,0 +1,130 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "datanyx"
+version = "0.1.0"
+description = "Turn sensitive data into a value-free schema that AI agents can work from."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["ai-agents", "privacy", "schema", "anonymization", "data"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Security",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = ["sqlalchemy>=2", "tomli>=2; python_version < '3.11'"]
+
+[project.optional-dependencies]
+postgres = ["psycopg[binary]>=3.1"]
+mysql = ["pymysql>=1.1", "cryptography"]
+mssql = ["pymssql>=2.3"]
+oracle = ["oracledb>=2"]
+parquet = ["pyarrow>=14"]
+excel = ["openpyxl>=3.1"]
+yaml = ["pyyaml>=6"]
+streaming = ["ijson>=3.2"]
+mcp = ["mcp>=1.2"]
+avro = ["fastavro>=1.9"]
+orc = ["pyarrow>=14"]
+duckdb = ["duckdb>=1.0"]
+# pyreadstat publishes no Windows wheel for Python 3.10, and building it there needs a C library, so
+# the extra is empty on that one platform and SAS, Stata and SPSS files report that they need it.
+stats = ["pyreadstat>=1.2; sys_platform != 'win32' or python_version >= '3.11'"]
+mongodb = ["pymongo>=4.6"]
+all = ["datanyx[postgres,mysql,mssql,oracle,parquet,excel,yaml,streaming,mcp,check,pdf,presidio,avro,orc,duckdb,stats,mongodb]"]
+text = ["spacy>=3.7"]
+check = [
+    "sqlglot>=25",
+]
+pdf = [
+    "pypdf>=4",
+]
+presidio = [
+    "presidio-analyzer>=2.2",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8",
+    "pytest-cov>=5",
+    "hypothesis>=6",
+    "ruff>=0.6",
+    "mypy>=1.10",
+    "types-PyYAML",
+    "pandas>=2.3.3",
+    "mongomock>=4.1",
+]
+docs = [
+    "mkdocs>=1.6",
+    "mkdocs-material>=9.5",
+]
+# Large NER models, installed from spaCy's releases. Needed only by the text tests.
+models = [
+    "en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl",
+    "fr-core-news-md @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.8.0/fr_core_news_md-3.8.0-py3-none-any.whl",
+]
+
+[project.scripts]
+datanyx = "datanyx.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+markers = [
+    "server: needs a real database server, started from tests/docker-compose.yml",
+    "needs_model: runs the default spaCy model, which only some CI jobs install",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
+ignore = [
+    "E501", "B904", "SIM105", "RUF012",
+    "B905",    # zip() on rows of unequal length is intended: short rows leave columns out.
+    "RUF001",  # Typographic apostrophes and dotless i are test data and word lists.
+    "SIM115",  # Streams that must outlive a `with` block (stdin, SQLite handles).
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["B017", "B028", "SIM117"]
+
+[tool.mypy]
+python_version = "3.10"
+files = ["src/datanyx"]
+ignore_missing_imports = true
+warn_unused_ignores = true
+check_untyped_defs = true
+
+[tool.coverage.run]
+source = ["src/datanyx"]
+branch = true
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
+[[tool.mypy.overrides]]
+module = "datanyx.mcp_server"
+# Which of the two MCP library versions is installed decides which ignores are needed.
+warn_unused_ignores = false
+
+[tool.uv]
+default-groups = ["dev"]

datanyx-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

datanyx-0.1.0/src/datanyx/__init__.py

@@ -0,0 +1,117 @@
+"""DataNyx: describe data to an AI agent without showing it the data."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from dataclasses import replace
+
+from .keys import Evidence, apply_evidence
+from .privacy import KeyPolicy
+from .renderers import RENDERERS
+from .schema import DataSchema
+from .sources import Source, SourceError, describe_folder, resolve_source
+from .text import DEFAULT_MODEL
+
+__version__ = "0.1.0"
+
+
+def extract(
+    target: str,
+    source: str | None = None,
+    policy: KeyPolicy | None = None,
+    namespace: str | None = None,
+    recursive: bool = False,
+) -> DataSchema:
+    """Build a value-free schema for a file path, "-" for stdin, a database URL or a folder.
+
+    `namespace` picks a database schema; the database's default is used otherwise.
+    For a folder, every readable file in it is described (and in its subfolders,
+    when `recursive` is set); `source` then limits which files are read.
+    """
+    policy = policy or KeyPolicy()
+    if policy.infer_keys and policy.evidence is None:
+        policy = replace(policy, evidence=Evidence())
+    if os.path.isdir(target):
+        if namespace is not None:
+            raise SourceError("a schema can only be selected for SQL databases")
+        schema = describe_folder(target, source, policy, recursive)
+    else:
+        schema = resolve_source(target, source).extract(target, policy, namespace)
+    if policy.infer_keys and policy.evidence is not None:
+        schema = apply_evidence(schema, policy.evidence)
+        if not any(e.foreign_keys or _has_unique(e) for e in schema.entities):
+            schema = replace(schema, notes=(*schema.notes, "no unique columns or joins were inferred; this kind of source may not support it, or no key-looking column qualified"))
+    if policy.scale_hints and not any(entity.scale for entity in schema.entities):
+        schema = replace(schema, notes=(*schema.notes, "no row-count hints: this kind of source can't give one without reading it"))
+    return schema
+
+
+def _has_unique(entity) -> bool:
+    node = getattr(entity.node, "items", entity.node)
+    return any(field.unique for field in getattr(node, "fields", {}).values())
+
+
+def render(schema: DataSchema, output_format: str = "json-schema") -> str:
+    """Render a schema in one of the formats listed in `RENDERERS`."""
+    return RENDERERS[output_format](schema)
+
+
+def schema_of(
+    target: str,
+    output_format: str = "json-schema",
+    *,
+    source: str | None = None,
+    map_paths: Iterable[str] = (),
+    keep_names: Iterable[str] = (),
+    namespace: str | None = None,
+    recursive: bool = False,
+) -> str:
+    """The schema of `target` as text, in one call.
+
+    >>> print(datanyx.schema_of("customers.csv", "text"))
+
+    Raises `SourceError` with a message that never quotes the data. Use `extract` and
+    `render` instead to get the schema as objects, or to see its `notes`.
+    """
+    policy = KeyPolicy(map_paths=frozenset(map_paths), keep_names=frozenset(keep_names))
+    schema = extract(target, source, policy, namespace, recursive)
+    return render(schema, output_format)
+
+
+def anonymize_text(
+    text: str,
+    *,
+    model: str = DEFAULT_MODEL,
+    hide: Iterable[str] = (),
+    keep: Iterable[str] = (),
+    keep_entity: Iterable[str] = (),
+) -> str:
+    """`text` with names and identifiers replaced by placeholders such as <PERSON_1>.
+
+    Needs `datanyx[text]` and a spaCy model. Best effort: review the result.
+    """
+    from .text import Detector, PatternDetector, SpacyDetector, anonymize
+
+    detectors: list[Detector] = [PatternDetector(), SpacyDetector(model)]
+    return anonymize(text, detectors, hide, keep, keep_entity)
+
+
+def synthesize(schema: DataSchema, rows: int = 100, seed: int | None = 0):
+    """Fake data matching `schema`, as a list of `datanyx.synth.Table`. See `datanyx.synth`."""
+    from .synth import synthesize as generate
+
+    return generate(schema, rows, seed)
+
+
+__all__ = [
+    "DataSchema",
+    "KeyPolicy",
+    "Source",
+    "SourceError",
+    "anonymize_text",
+    "extract",
+    "render",
+    "schema_of",
+    "synthesize",
+]

datanyx-0.1.0/src/datanyx/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())