datacom 0.2.0__py3-none-win_arm64.whl

datacom/__init__.py

@@ -0,0 +1,131 @@
+"""Clean, profile and query messy data files from Python.
+
+A thin wrapper around the datacom binary that ships inside this package:
+every function runs the same engine as the command line, so results match.
+"""
+
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+
+__all__ = ["DatacomError", "binary", "run", "profile", "clean", "query", "convert", "clean_frame", "__version__"]
+__version__ = "0.2.0"
+
+_REPEATABLE = {"where", "derive"}
+
+
+class DatacomError(RuntimeError):
+    """datacom exited with an error; stderr holds its message."""
+
+    def __init__(self, returncode, stderr):
+        super().__init__(stderr or f"datacom exited with status {returncode}")
+        self.returncode = returncode
+        self.stderr = stderr
+
+
+def binary():
+    """Path of the datacom executable: DATACOM_BINARY, else the bundled one."""
+    env = os.environ.get("DATACOM_BINARY")
+    if env:
+        return env
+    exe = Path(__file__).parent / "bin" / ("datacom.exe" if os.name == "nt" else "datacom")
+    if not exe.exists():
+        raise FileNotFoundError(f"no datacom binary at {exe}; install a platform wheel or set DATACOM_BINARY")
+    return str(exe)
+
+
+def _flags(options):
+    args = []
+    for key, value in options.items():
+        if value is None or value is False:
+            continue
+        flag = "-" + key.replace("_", "-")
+        if value is True:
+            args.append(flag)
+        elif isinstance(value, (list, tuple)):
+            if key in _REPEATABLE:
+                for v in value:
+                    args += [flag, str(v)]
+            else:
+                args += [flag, ",".join(str(v) for v in value)]
+        else:
+            args += [flag, str(value)]
+    return args
+
+
+def _sources(source):
+    if isinstance(source, (str, os.PathLike)):
+        return [os.fspath(source)]
+    return [os.fspath(s) for s in source]
+
+
+def run(command, *sources, **options):
+    """Run any datacom command and return the CompletedProcess.
+
+    Keyword options become flags: snake=True is -snake, near_dupes=0.8 is
+    -near-dupes 0.8, text=["title", "body"] is -text title,body, and lists
+    for where and derive repeat the flag.
+    """
+    args = [binary(), command, *(os.fspath(s) for s in sources), *_flags(options)]
+    p = subprocess.run(args, capture_output=True, text=True)
+    if p.returncode != 0:
+        raise DatacomError(p.returncode, p.stderr.strip())
+    return p
+
+
+def profile(source, **options):
+    """Summarize a file and return the report as a dict."""
+    return json.loads(run("profile", *_sources(source), json=True, **options).stdout)
+
+
+def _write(command, source, output, options):
+    with tempfile.TemporaryDirectory() as d:
+        manifest = options.pop("manifest", None) or os.path.join(d, "run.json")
+        run(command, *_sources(source), o=os.fspath(output), manifest=manifest, q=True, **options)
+        with open(manifest, encoding="utf-8") as f:
+            return json.load(f)
+
+
+def clean(source, output, **options):
+    """Clean source into output and return the run manifest as a dict.
+
+    The manifest lists input and output SHA-256, every rule applied and how
+    many rows or values each one dropped or changed.
+    """
+    return _write("clean", source, output, options)
+
+
+def query(source, output, **options):
+    """Filter, derive, join, group or sort, and return the run manifest."""
+    return _write("query", source, output, options)
+
+
+def convert(source, output, **options):
+    """Write source in the format output's extension names; returns the manifest."""
+    return _write("convert", source, output, options)
+
+
+def clean_frame(df, **options):
+    """Clean a pandas DataFrame and return a new one.
+
+    The frame goes through a temporary Parquet file when pyarrow is
+    installed, and CSV otherwise.
+    """
+    import pandas as pd
+
+    try:
+        import pyarrow  # noqa: F401
+
+        ext = ".parquet"
+    except ImportError:
+        ext = ".csv"
+    with tempfile.TemporaryDirectory() as d:
+        src, dst = os.path.join(d, "in" + ext), os.path.join(d, "out" + ext)
+        if ext == ".parquet":
+            df.to_parquet(src, index=False)
+        else:
+            df.to_csv(src, index=False)
+        clean(src, dst, **options)
+        return pd.read_parquet(dst) if ext == ".parquet" else pd.read_csv(dst, dtype=str, keep_default_na=False)

datacom/__main__.py

@@ -0,0 +1,16 @@
+import os
+import subprocess
+import sys
+
+from . import binary
+
+
+def main():
+    exe = binary()
+    if os.name == "nt":
+        sys.exit(subprocess.call([exe, *sys.argv[1:]]))
+    os.execv(exe, [exe, *sys.argv[1:]])
+
+
+if __name__ == "__main__":
+    main()

datacom-0.2.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 datacom authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datacom-0.2.0.dist-info/METADATA

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: datacom
+Version: 0.2.0
+Summary: Clean, profile and query messy CSV, JSON, Excel and Parquet files; mask personal data and dedupe text for AI training
+Home-page: https://github.com/moodykhalif23/datacom
+Project-URL: Source, https://github.com/moodykhalif23/datacom
+License: MIT
+Requires-Python: >=3.8
+Provides-Extra: pandas
+Requires-Dist: pandas; extra == 'pandas'
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Description-Content-Type: text/markdown
+
+# datacom for Python
+
+Clean, profile and query messy data files, and prepare text for AI training, from Python. The package bundles the datacom binary for your platform, so there is nothing else to install and the results match the command line exactly.
+
+```sh
+pip install datacom
+```
+
+## Quick start
+
+```python
+import datacom
+
+report = datacom.profile("sales.xlsx")
+print(report["rows"], report["issues"])
+
+manifest = datacom.clean("raw.csv.gz", "clean.parquet", snake=True, dedupe=True, redact="all")
+print(manifest["rows_out"], manifest["outputs"][0]["sha256"])
+```
+
+`clean`, `query` and `convert` return the run manifest: input and output SHA-256, every rule applied, and how many rows or values each one dropped or changed.
+
+## Training data
+
+```python
+manifest = datacom.clean(
+    "s3://lake/crawl/2026-11/*.jsonl.gz",
+    "clean/part-*.parquet",
+    text=["body"],
+    quality="default",
+    decontam="evals/mmlu.jsonl",
+    near_dupes=0.8,
+    redact="all",
+    workers=16,
+)
+for rule in manifest["rules"]:
+    print(rule)
+```
+
+- `text` cleans those columns as documents: repairs garbled characters, strips HTML, normalizes Unicode and removes control characters.
+- `quality` applies the Gopher rules and counts what each rule dropped.
+- `decontam` drops documents that share a 13-word sequence with a test set.
+- `near_dupes` removes near-duplicates with MinHash in a fixed-size index.
+- `redact` masks emails, phone numbers, ID numbers and card numbers.
+
+Sources can be local paths, URLs, `s3://`, `gs://` or `hf://datasets/...`, with globs. Credentials come from the usual places: `AWS_*` variables or `~/.aws/credentials`, `GOOGLE_OAUTH_ACCESS_TOKEN`, and `HF_TOKEN`.
+
+## Options
+
+Keyword arguments become flags: `snake=True` is `-snake`, `near_dupes=0.8` is `-near-dupes 0.8`, `text=["title", "body"]` is `-text title,body`, and lists for `where` and `derive` repeat the flag. `datacom.run(command, *sources, **options)` runs any command and returns the `subprocess.CompletedProcess`; failures raise `datacom.DatacomError` with datacom's message.
+
+With pandas installed, `datacom.clean_frame(df, snake=True, dedupe=True)` returns a cleaned copy of a DataFrame.
+
+The `datacom` command is installed too, so everything in the [main documentation](https://github.com/moodykhalif23/datacom) works from the shell.

datacom-0.2.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+datacom/__init__.py,sha256=p9JuVjkYJGZ74ZdBmRVaU721Ty2nn9XgehjxFhAzzfA,4343
+datacom/__main__.py,sha256=WhCuPfMyoVb33x6lbmw3nJ0pPgE6AjC-z68wMbGlxVA,254
+datacom/bin/datacom.exe,sha256=ymEbKFvWBz_e72ZBSfky1_7pEOwmeP2YyjsjW2fReLQ,13243904
+datacom-0.2.0.dist-info/METADATA,sha256=7n6cAxvwqHScEyZjDoBS2CzuinCuNMObKfwvi1YwZXE,2922
+datacom-0.2.0.dist-info/WHEEL,sha256=WSv1FTpHVEz5pEOm_F7Jn0CggS7qvoB_Mp_8uTeH84U,101
+datacom-0.2.0.dist-info/entry_points.txt,sha256=jE44ePB7nafDdB9oPRlP-vvk1xVSImhYf_HzE5MCx9c,50
+datacom-0.2.0.dist-info/LICENSE,sha256=hiacSLWN2WAlnrLY6Z9unO8_NJveZy-8mSRf0BkrPqw,1072
+datacom-0.2.0.dist-info/RECORD,,

datacom-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: datacom tools/wheels.py
+Root-Is-Purelib: false
+Tag: py3-none-win_arm64

datacom-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+datacom = datacom.__main__:main