classifyre-cli 0.7.2__tar.gz → 0.7.4__tar.gz

classifyre_cli-0.7.4/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 341 packages in 545ms
+Checked 52 packages in 1ms

{classifyre_cli-0.7.2 → classifyre_cli-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.7.2
+Version: 0.7.4
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.7.2 → classifyre_cli-0.7.4}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.7.2 → classifyre_cli-0.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.7.2"
+version = "0.7.4"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably
@@ -25,7 +25,10 @@ dependencies = [
 [tool.uv.sources]
 # Local editable install for development — ignored when published to PyPI
 classifyre-schemas = { path = "../../packages/schemas", editable = true }
-en-core-web-sm = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }
+# Explosion's en_core_web_sm 3.8.0 wheel, unchanged (same sha256), from our own
+# Hugging Face copy pinned to a commit: GitHub's release download answered 504
+# for hours and failed every CI run and image build that needed it.
+en-core-web-sm = { url = "https://huggingface.co/classifyre/en_core_web_sm/resolve/36818c1a75b5bac93b4f0df26b76da8d1149ce8c/en_core_web_sm-3.8.0-py3-none-any.whl" }
 # CPU-only torch on Linux. The default PyPI Linux wheels bundle the full NVIDIA
 # CUDA stack (~5-7 GB) which is dead weight on our CPU-only CLI jobs (no GPU
 # requested). Pull torch/torchvision from the PyTorch CPU index on Linux so the
@@ -170,10 +173,22 @@ file-processing = [
     # when none is installed.
     "py7zr>=1.1.3",
     "rarfile>=4.3",
-    # Table layer: Iceberg metadata resolution + SQL over Parquet without Spark.
-    "pyiceberg>=0.9.0",
+    # Table layer: CSV/TSV, JSON lines, Avro, database files and the Delta and
+    # Iceberg tables bucket sources find (utils/table_files, utils/lakehouse_tables).
     "duckdb>=1.1.0",
 ]
+# Lakehouse table formats DuckDB does not read reliably: Lance (pylance applies
+# deletes and takes object_store storage options), Hudi (hudi-rs snapshot reads,
+# copy-on-write and merge-on-read) and Paimon (pypaimon merges primary-key
+# tables; DuckDB's community extension cannot take storage credentials). Delta
+# and Iceberg go through DuckDB in file-processing. Installed on first use, when
+# a bucket scan finds a table in one of these formats.
+lakehouse = [
+    { include-group = "file-processing" },
+    "pylance>=0.30.0",
+    "hudi>=0.5.0",
+    "pypaimon>=2.1.0",
+]
 transcription = [
     # CPU audio/video transcription. faster-whisper decodes media via bundled
     # PyAV (no system ffmpeg required) and runs ctranslate2 for inference.
@@ -208,7 +223,7 @@ mssql = [
     "pymssql>=2.3.4,<3.0.0",
 ]
 oracle = [
-    "oracledb>=2.5.1,<5.0.0",
+    "oracledb>=2.5.1,<27.0.0",
 ]
 hive = [
     "pyhive>=0.7.0,<1.0.0",
@@ -221,7 +236,7 @@ databricks = [
     "pyarrow>=24.0.0",
 ]
 snowflake = [
-    "snowflake-connector-python>=4.5.0,<5.0.0",
+    "snowflake-connector-python>=4.5.0,<6.0.0",
 ]
 mongodb = [
     "pymongo>=4.17.0,<5.0.0",
@@ -279,19 +294,29 @@ reddit = [
     "yt-dlp>=2025.1.0",
     "youtube-transcript-api>=1.0.0",
 ]
-delta-lake = [
-    # Pure-Python/Rust Delta reader (delta-rs) — no Spark, no JVM. Table metadata
-    # and data-file lists come from deltalake; rows are sampled through DuckDB
-    # (file-processing group) over the table's Parquet files.
-    { include-group = "file-processing" },
-    "deltalake>=0.25.0",
+trino = [
+    # Trino's own DB-API client; also Starburst Galaxy and Enterprise.
+    "trino>=0.330.0,<1.0.0",
+]
+athena = [
+    # Athena SQL (pyathena) and the Glue Data Catalog (boto3).
+    "pyathena>=3.9.0,<4.0.0",
     "boto3>=1.43.10,<2.0.0",
 ]
-iceberg = [
-    # PyIceberg resolves table metadata and finds Parquet files without Spark;
-    # DuckDB (file-processing group) samples rows from those files.
+bigquery = [
+    # BigQuery client: tabledata.list for free row reads, queries only for
+    # RANDOM/LATEST samples.
+    "google-cloud-bigquery>=3.27.0,<4.0.0",
+]
+redshift = [
+    # Amazon's driver: password and IAM sign-in (provisioned and Serverless).
+    "redshift-connector>=2.1.0,<3.0.0",
+]
+iceberg-rest = [
+    # DuckDB attaches the catalog and reads its tables (file-processing);
+    # google-auth mints the access token for the BigLake metastore.
     { include-group = "file-processing" },
-    "boto3>=1.43.10,<2.0.0",
+    "google-auth>=2.40,<3.0",
 ]
 kafka = [
     # librdkafka-based client (bundled wheels, no JVM).
@@ -422,8 +447,9 @@ module = [
     "re2.*",
     "faster_whisper",
     "faster_whisper.*",
-    "pyiceberg.*",
-    "deltalake.*",
+    "lance.*",
+    "hudi.*",
+    "pypaimon.*",
     "duckdb.*",
     "confluent_kafka.*",
     "boto3.*",
@@ -483,6 +509,11 @@ package = true
 # distribution declared by our video and OCR groups. Installing both would
 # overwrite the same cv2 namespace and add GUI libraries to CPU-only containers.
 exclude-dependencies = ["opencv-python"]
+# pypaimon declares pyarrow<20, a pin from before newer pyarrow existed rather
+# than a known break: its reader works on the pyarrow the rest of the CLI uses
+# (the Databricks group needs >=24), which tests/utils/test_lakehouse_tables.py
+# exercises. Without the override no environment can hold both groups.
+override-dependencies = ["pyarrow>=24.0.0"]
 # Two transitive dependencies whose latest release breaks the package that
 # pulls it in, while that package's own requirement still allows it. Both
 # failures were silent in a scan: images and scans came back with no text, and

{classifyre_cli-0.7.2 → classifyre_cli-0.7.4}/scripts/generate_runtime_packages.py

@@ -78,8 +78,12 @@ INCLUDED_GROUPS = (
     "youtube",
     "reddit",
     "kafka",
-    "delta-lake",
-    "iceberg",
+    "lakehouse",
+    "iceberg-rest",
+    "trino",
+    "athena",
+    "bigquery",
+    "redshift",
     # sqlglot: pure Python and small, and the thing a connector author reaches
     # for when they want to derive column lineage from a query they already have.
     "lineage",
@@ -125,6 +129,8 @@ MODULE_ALIASES: dict[str, tuple[str, ...]] = {
     "google-api-python-client": ("googleapiclient",),
     "google-auth": ("google.auth",),
     "google-cloud-storage": ("google.cloud.storage",),
+    "google-cloud-bigquery": ("google.cloud.bigquery",),
+    "pylance": ("lance",),
     "msgraph-sdk": ("msgraph",),
     "en-core-web-sm": ("en_core_web_sm",),
     "classifyre-schemas": ("schemas",),

{classifyre_cli-0.7.2 → classifyre_cli-0.7.4}/src/detectors/pii/detector.py

@@ -151,6 +151,9 @@ class PIIDetector(BaseDetector):
         "fax": {"PHONE_NUMBER"},
         "name": {"PERSON"},
         "person": {"PERSON"},
+        # A "contact" column holds whoever to reach and how: a name, an e-mail
+        # address or a phone number, often one of each across rows.
+        "contact": {"PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER"},
         "address": {"LOCATION"},
         "location": {"LOCATION"},
         "city": {"LOCATION"},