classifyre-cli 0.7.1__tar.gz → 0.7.3__tar.gz

classifyre_cli-0.7.3/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 338 packages in 176ms
+Checked 52 packages in 1ms

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.7.1
+Version: 0.7.3
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.7.1"
+version = "0.7.3"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably
@@ -170,10 +170,22 @@ file-processing = [
     # when none is installed.
     "py7zr>=1.1.3",
     "rarfile>=4.3",
-    # Table layer: Iceberg metadata resolution + SQL over Parquet without Spark.
-    "pyiceberg>=0.9.0",
+    # Table layer: CSV/TSV, JSON lines, Avro, database files and the Delta and
+    # Iceberg tables bucket sources find (utils/table_files, utils/lakehouse_tables).
     "duckdb>=1.1.0",
 ]
+# Lakehouse table formats DuckDB does not read reliably: Lance (pylance applies
+# deletes and takes object_store storage options), Hudi (hudi-rs snapshot reads,
+# copy-on-write and merge-on-read) and Paimon (pypaimon merges primary-key
+# tables; DuckDB's community extension cannot take storage credentials). Delta
+# and Iceberg go through DuckDB in file-processing. Installed on first use, when
+# a bucket scan finds a table in one of these formats.
+lakehouse = [
+    { include-group = "file-processing" },
+    "pylance>=0.30.0",
+    "hudi>=0.5.0",
+    "pypaimon>=2.1.0",
+]
 transcription = [
     # CPU audio/video transcription. faster-whisper decodes media via bundled
     # PyAV (no system ffmpeg required) and runs ctranslate2 for inference.
@@ -208,7 +220,7 @@ mssql = [
     "pymssql>=2.3.4,<3.0.0",
 ]
 oracle = [
-    "oracledb>=2.5.1,<5.0.0",
+    "oracledb>=2.5.1,<27.0.0",
 ]
 hive = [
     "pyhive>=0.7.0,<1.0.0",
@@ -221,7 +233,7 @@ databricks = [
     "pyarrow>=24.0.0",
 ]
 snowflake = [
-    "snowflake-connector-python>=4.5.0,<5.0.0",
+    "snowflake-connector-python>=4.5.0,<6.0.0",
 ]
 mongodb = [
     "pymongo>=4.17.0,<5.0.0",
@@ -279,19 +291,29 @@ reddit = [
     "yt-dlp>=2025.1.0",
     "youtube-transcript-api>=1.0.0",
 ]
-delta-lake = [
-    # Pure-Python/Rust Delta reader (delta-rs) — no Spark, no JVM. Table metadata
-    # and data-file lists come from deltalake; rows are sampled through DuckDB
-    # (file-processing group) over the table's Parquet files.
-    { include-group = "file-processing" },
-    "deltalake>=0.25.0",
+trino = [
+    # Trino's own DB-API client; also Starburst Galaxy and Enterprise.
+    "trino>=0.330.0,<1.0.0",
+]
+athena = [
+    # Athena SQL (pyathena) and the Glue Data Catalog (boto3).
+    "pyathena>=3.9.0,<4.0.0",
     "boto3>=1.43.10,<2.0.0",
 ]
-iceberg = [
-    # PyIceberg resolves table metadata and finds Parquet files without Spark;
-    # DuckDB (file-processing group) samples rows from those files.
+bigquery = [
+    # BigQuery client: tabledata.list for free row reads, queries only for
+    # RANDOM/LATEST samples.
+    "google-cloud-bigquery>=3.27.0,<4.0.0",
+]
+redshift = [
+    # Amazon's driver: password and IAM sign-in (provisioned and Serverless).
+    "redshift-connector>=2.1.0,<3.0.0",
+]
+iceberg-rest = [
+    # DuckDB attaches the catalog and reads its tables (file-processing);
+    # google-auth mints the access token for the BigLake metastore.
     { include-group = "file-processing" },
-    "boto3>=1.43.10,<2.0.0",
+    "google-auth>=2.40,<3.0",
 ]
 kafka = [
     # librdkafka-based client (bundled wheels, no JVM).
@@ -422,8 +444,9 @@ module = [
     "re2.*",
     "faster_whisper",
     "faster_whisper.*",
-    "pyiceberg.*",
-    "deltalake.*",
+    "lance.*",
+    "hudi.*",
+    "pypaimon.*",
     "duckdb.*",
     "confluent_kafka.*",
     "boto3.*",
@@ -483,6 +506,11 @@ package = true
 # distribution declared by our video and OCR groups. Installing both would
 # overwrite the same cv2 namespace and add GUI libraries to CPU-only containers.
 exclude-dependencies = ["opencv-python"]
+# pypaimon declares pyarrow<20, a pin from before newer pyarrow existed rather
+# than a known break: its reader works on the pyarrow the rest of the CLI uses
+# (the Databricks group needs >=24), which tests/utils/test_lakehouse_tables.py
+# exercises. Without the override no environment can hold both groups.
+override-dependencies = ["pyarrow>=24.0.0"]
 # Two transitive dependencies whose latest release breaks the package that
 # pulls it in, while that package's own requirement still allows it. Both
 # failures were silent in a scan: images and scans came back with no text, and

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/scripts/generate_runtime_packages.py

@@ -78,8 +78,12 @@ INCLUDED_GROUPS = (
     "youtube",
     "reddit",
     "kafka",
-    "delta-lake",
-    "iceberg",
+    "lakehouse",
+    "iceberg-rest",
+    "trino",
+    "athena",
+    "bigquery",
+    "redshift",
     # sqlglot: pure Python and small, and the thing a connector author reaches
     # for when they want to derive column lineage from a query they already have.
     "lineage",
@@ -125,6 +129,8 @@ MODULE_ALIASES: dict[str, tuple[str, ...]] = {
     "google-api-python-client": ("googleapiclient",),
     "google-auth": ("google.auth",),
     "google-cloud-storage": ("google.cloud.storage",),
+    "google-cloud-bigquery": ("google.cloud.bigquery",),
+    "pylance": ("lance",),
     "msgraph-sdk": ("msgraph",),
     "en-core-web-sm": ("en_core_web_sm",),
     "classifyre-schemas": ("schemas",),

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/src/detectors/pii/detector.py

@@ -151,6 +151,9 @@ class PIIDetector(BaseDetector):
         "fax": {"PHONE_NUMBER"},
         "name": {"PERSON"},
         "person": {"PERSON"},
+        # A "contact" column holds whoever to reach and how: a name, an e-mail
+        # address or a phone number, often one of each across rows.
+        "contact": {"PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER"},
         "address": {"LOCATION"},
         "location": {"LOCATION"},
         "city": {"LOCATION"},

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/src/graph/edges.py

@@ -131,6 +131,7 @@ class FieldTransform(StrEnum):
     IDENTITY = "IDENTITY"  # copied through unchanged
     TRANSFORMED = "TRANSFORMED"  # an expression combined the upstreams
     AGGREGATED = "AGGREGATED"  # SUM/COUNT/... over a group
+    MAPPED = "MAPPED"  # the platform says it feeds this column, not how
     INDIRECT = "INDIRECT"  # influenced the rows, not the values
 
 

{classifyre_cli-0.7.1 → classifyre_cli-0.7.3}/src/main.py

@@ -976,6 +976,9 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     ):
                         sink.set_listing_complete()
 
+                    if hasattr(sink, "set_sweep_complete"):
+                        sink.set_sweep_complete(getattr(source, "sweep_complete", None))
+
                     # Relationships go *before* finish(): finish() finalizes the
                     # run and marks it COMPLETED, and edges that land after that
                     # belong to a run that already claimed to be done.