classifyre-cli 0.6.7__tar.gz → 0.6.9__tar.gz

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 325 packages in 264ms
+Resolved 325 packages in 114ms
 Checked 51 packages in 6ms

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.7
+Version: 0.6.9
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.7",
+  "version": "0.6.9",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.7"
+version = "0.6.9"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/src/main.py

@@ -909,6 +909,8 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     # completion path — a timed-out run must not advance it.
                     if hasattr(sink, "set_sampling_cursor"):
                         sink.set_sampling_cursor(source.current_sampling_cursor())
+                    if hasattr(sink, "set_sampling_fingerprint"):
+                        sink.set_sampling_fingerprint(source.sampling_fingerprint())
 
                     # A connector that covered a slice must say so before
                     # finish(), which is what decides whether absence retires

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/src/models/generated_input.py

@@ -35,6 +35,7 @@ class AssetType(StrEnum):
     HIVE = 'HIVE'
     DATABRICKS = 'DATABRICKS'
     SNOWFLAKE = 'SNOWFLAKE'
+    DREMIO = 'DREMIO'
     MONGODB = 'MONGODB'
     NEO4J = 'NEO4J'
     POWERBI = 'POWERBI'
@@ -101,7 +102,7 @@ class SlackChannelType(StrEnum):
 
 class SamplingStrategy(StrEnum):
     """
-    Sampling strategy. AUTOMATIC (recommended default) incrementally ingests a new slice of not-yet-seen data on every run, remembering its position between runs and wrapping around to re-scan from the start once everything has been covered — eventually ingesting everything at a bounded cost per run. RANDOM samples items randomly. LATEST prioritises the most recently modified/created items. ALL scans every item with no limit.
+    Sampling strategy. AUTOMATIC (recommended default) reads a bounded slice on every run and remembers where it has been: whatever is new since the last run comes first, the rest of the slice continues backwards through older data, and once everything has been covered only new data is read. Changing the source's settings or one of its detectors starts the sweep over. A table or collection with no increasing key cannot tell new rows from old, so it is paged from the start and wraps around instead. RANDOM samples items randomly. LATEST prioritises the most recently modified/created items. ALL scans every item with no limit.
     """
 
     AUTOMATIC = 'AUTOMATIC'
@@ -368,6 +369,7 @@ class Type(StrEnum):
     HIVE = 'HIVE'
     DATABRICKS = 'DATABRICKS'
     SNOWFLAKE = 'SNOWFLAKE'
+    DREMIO = 'DREMIO'
     MONGODB = 'MONGODB'
     NEO4J = 'NEO4J'
     POWERBI = 'POWERBI'
@@ -2152,6 +2154,150 @@ class SnowflakeOptional(BaseModel):
     extraction: SnowflakeOptionalExtraction | None = None
 
 
+class DremioCloudRegion(StrEnum):
+    """
+    Dremio Cloud control plane the project lives in
+    """
+
+    US = 'US'
+    EU = 'EU'
+
+
+class DremioRequiredSoftware(BaseModel):
+    """
+    A Dremio you run yourself (Dremio Software, Enterprise or Community Edition).
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    url: AnyUrl = Field(
+        ...,
+        description='Address you open Dremio at, including the port (for example, https://dremio.example.com:9047)',
+    )
+
+
+class DremioRequiredCloud(BaseModel):
+    """
+    A project in Dremio's hosted service. Signs in with a personal access token.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    project_id: str = Field(
+        ...,
+        description='Dremio Cloud project ID (Project Settings > General Information)',
+    )
+    region: DremioCloudRegion
+
+
+class DremioMaskedPassword(BaseModel):
+    """
+    Self-hosted Dremio only.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    username: str = Field(..., description='Dremio login username')
+    password: str = Field(..., description='Dremio login password')
+
+
+class DremioMaskedToken(BaseModel):
+    """
+    Works for Dremio Cloud and for self-hosted Enterprise Edition.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    token: str = Field(
+        ...,
+        description='Dremio personal access token (Account Settings > Personal Access Tokens)',
+    )
+
+
+class DremioOptionalConnection(BaseModel):
+    """
+    Dremio API and query tuning options.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    verify_ssl: bool | None = Field(
+        True,
+        description="Verify the Dremio server's TLS certificate. Turn off only for a self-hosted Dremio with a self-signed certificate.",
+    )
+    timeout_seconds: int | None = Field(
+        30, description='HTTP timeout for a single Dremio API call', ge=5, le=300
+    )
+    query_timeout_seconds: int | None = Field(
+        300,
+        description='Maximum time to wait for one sampling query to finish before it is cancelled',
+        ge=10,
+        le=3600,
+    )
+
+
+class DremioOptionalScope(BaseModel):
+    """
+    Which Dremio sources, spaces, folders, and datasets to scan.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_paths: list[str] | None = Field(
+        None,
+        description='Optional allowlist of sources, spaces, folders, or datasets, written as dot-separated paths (for example, Marketing or Samples."samples.dremio.com"). Everything beneath a listed path is scanned. Wrap a name that itself contains a dot in double quotes.',
+    )
+    exclude_paths: list[str] | None = Field(
+        [],
+        description='Sources, spaces, folders, or datasets to skip, written the same way as include_paths. Exclusions win over inclusions.',
+    )
+    include_tables: bool | None = Field(
+        True, description='Include tables (physical datasets) in extraction'
+    )
+    include_views: bool | None = Field(
+        True, description='Include views (virtual datasets) in extraction'
+    )
+    include_home_spaces: bool | None = Field(
+        False, description='Include personal home spaces (@username) in extraction'
+    )
+    table_limit: int | None = Field(
+        None, description='Optional cap on number of table/view assets extracted', ge=1
+    )
+
+
+class DremioOptionalExtraction(BaseModel):
+    """
+    Lineage extraction controls for Dremio.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_view_lineage: bool | None = Field(
+        True,
+        description="Link each view to the tables and views it reads from. Uses Dremio's own lineage graph where the edition provides it, and the view's SQL otherwise.",
+    )
+    include_source_lineage: bool | None = Field(
+        True,
+        description="Link tables that Dremio reads from an external database (PostgreSQL, MySQL, SQL Server, Oracle, Snowflake) to that database's own table, so lineage continues across systems. Needs permission to view the Dremio source's settings.",
+    )
+
+
+class DremioOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    connection: DremioOptionalConnection | None = None
+    scope: DremioOptionalScope | None = None
+    extraction: DremioOptionalExtraction | None = None
+
+
 class MongoDBDeployment(StrEnum):
     """
     MongoDB deployment mode
@@ -2703,7 +2849,7 @@ class ConfluenceOptionalConnection(BaseModel):
     )
 
 
-class Type21(StrEnum):
+class Type22(StrEnum):
     """
     Filter spaces by space type
     """
@@ -2740,7 +2886,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
     keys: list[str] | None = Field(
         None, description='Filter spaces by keys (up to 250)', max_length=250
     )
-    type: Type21 | None = Field(None, description='Filter spaces by space type')
+    type: Type22 | None = Field(None, description='Filter spaces by space type')
     status: Status | None = Field(None, description='Filter spaces by status')
     labels: list[str] | None = Field(
         None,
@@ -3006,7 +3152,7 @@ class ServiceDeskOptional(BaseModel):
     content: ServiceDeskOptionalContent | None = None
 
 
-class Type22(StrEnum):
+class Type23(StrEnum):
     """
     Type of the asset or source
     """
@@ -3023,6 +3169,7 @@ class Type22(StrEnum):
     HIVE = 'HIVE'
     DATABRICKS = 'DATABRICKS'
     SNOWFLAKE = 'SNOWFLAKE'
+    DREMIO = 'DREMIO'
     MONGODB = 'MONGODB'
     NEO4J = 'NEO4J'
     POWERBI = 'POWERBI'
@@ -5183,6 +5330,30 @@ class SnowflakeInput(CoreInput):
     augmentation: AugmentationConfig | None = None
 
 
+class DremioInput(CoreInput):
+    type: Literal['DREMIO'] = Field(..., description='Type of the asset or source')
+    required: DremioRequiredSoftware | DremioRequiredCloud = Field(
+        ..., title='DremioRequired'
+    )
+    masked: DremioMaskedToken | DremioMaskedPassword = Field(..., title='DremioMasked')
+    optional: DremioOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    scan_cache: ScanCacheConfig | None = None
+    resources: ResourceOverrides | None = None
+    cleanup_removed_detector_findings: bool | None = Field(
+        True,
+        description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
+    )
+    augmentation: AugmentationConfig | None = None
+
+
 class MongoDBInput(CoreInput):
     type: Literal['MONGODB'] = Field(..., description='Type of the asset or source')
     required: MongoDBRequiredAtlas | MongoDBRequiredOnPrem = Field(
@@ -5848,6 +6019,7 @@ class SourceInput(
         | HiveInput
         | DatabricksInput
         | SnowflakeInput
+        | DremioInput
         | MongoDBInput
         | Neo4jInput
         | PowerBIInput
@@ -5889,6 +6061,7 @@ class SourceInput(
         | HiveInput
         | DatabricksInput
         | SnowflakeInput
+        | DremioInput
         | MongoDBInput
         | Neo4jInput
         | PowerBIInput

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/src/notebook/sdk.py

@@ -48,6 +48,7 @@ from ..graph.edges import (
     uses,
 )
 from ..utils.sampling_cursor import check_cursor_fits
+from ..utils.sampling_frontier import ListFrontier
 from ..utils.urn import Urn
 from .cohort import CohortItem, normalize_bands, select_cohort
 from .files import DEFAULT_PAGE_SIZE, ParsedContent, pages, parse
@@ -511,6 +512,7 @@ class Context:
         should_abort: Callable[[], bool] | None = None,
         query_assets: Callable[[dict[str, Any]], Any] | None = None,
         cohort_weights: Mapping[str, Mapping[str, float]] | None = None,
+        config_changed: bool = False,
     ) -> None:
         self._variables = dict(variables or {})
         self._secrets = dict(secrets or {})
@@ -535,6 +537,8 @@ class Context:
         }
         self._cohort_state: dict[str, Any] = {}
         self._cohort_stats: dict[str, Any] = {}
+        self._config_changed = bool(config_changed)
+        self._frontier_state: dict[str, Any] = {}
 
     # -- configuration -------------------------------------------------------
 
@@ -710,17 +714,97 @@ class Context:
         check_cursor_fits(cursor)
         self._next_cursor = dict(cursor)
 
+    @property
+    def config_changed(self) -> bool:
+        """True on the first run after this source's settings changed.
+
+        "Settings" is anything in the source's configuration -- scope,
+        sampling, variables, the notebook itself, which detectors are on --
+        and the definition of any custom detector it uses. Rotating a secret
+        does not count.
+
+        A built-in source starts its sweep over when that happens. A notebook
+        owns its own cursor, so nothing is taken from it; it is told instead,
+        and decides::
+
+            cursor = {} if ctx.config_changed else ctx.cursor
+
+        The flag stays set until a run finishes, so a run that fails before
+        acting on it does not lose it. ``ctx.newest_first()`` acts on it for
+        you.
+        """
+        return self._config_changed
+
+    def newest_first(
+        self,
+        items: Iterable[Any],
+        *,
+        order: Callable[[Any], tuple[Any, str]],
+        name: str = "items",
+        limit: int | None = None,
+    ) -> list[Any]:
+        """This run's slice of a listing: what is new first, then the backfill.
+
+        ::
+
+            def extract():
+                for doc in ctx.newest_first(
+                    api.list_documents(),
+                    order=lambda doc: (doc["updated_at"], doc["id"]),
+                ):
+                    yield Asset(id=doc["id"], ...)
+
+        Pass the whole listing, in any order. ``order`` maps an item to
+        ``(rank, id)``: ``rank`` is what makes an item newer -- a timestamp, an
+        increasing number -- and must be an int, float or string of one
+        consistent type that only changes when the item does; ``id`` is unique
+        to the item. Each run gets ``limit`` items (``rows_per_page`` by
+        default): everything newer than what earlier runs read comes first,
+        the rest continues backwards from where the last run stopped, and once
+        the whole listing has been read only new items are returned.
+
+        The position is kept in the cursor beside whatever you store there,
+        one per ``name``, and starts over when ``ctx.config_changed``. Calling
+        it declares this run's coverage partial.
+        """
+        stored = self._cursor.get("frontier")
+        previous = stored.get(str(name)) if isinstance(stored, dict) else None
+        frontier = ListFrontier(None if self._config_changed else previous)
+        window = frontier.window(
+            ((*order(item), item) for item in items),
+            int(limit) if limit else self.page_size,
+        )
+        state = frontier.to_state()
+        if state is not None:
+            self._frontier_state[str(name)] = state
+        if not self._partial_coverage:
+            self.set_partial_coverage(
+                f"the notebook reads a window ({name}) of a larger listing each run"
+            )
+        return window
+
     @property
     def next_cursor(self) -> dict[str, Any] | None:
-        if not self._cohort_state:
+        if not self._cohort_state and not self._frontier_state:
             return None if self._next_cursor is None else dict(self._next_cursor)
-        # A cohort's position is kept in the same cursor, beside whatever the
-        # notebook stores -- or, when it stores nothing this run, beside what
-        # the previous run left, which would otherwise be dropped.
+        # A cohort's or a frontier's position is kept in the same cursor,
+        # beside whatever the notebook stores -- or, when it stores nothing
+        # this run, beside what the previous run left, which would otherwise
+        # be dropped.
         base = self._next_cursor if self._next_cursor is not None else self._cursor
-        stored = base.get("cohort")
-        previous: dict[str, Any] = stored if isinstance(stored, dict) else {}
-        return {**base, "cohort": {**previous, **self._cohort_state}}
+        merged = dict(base)
+        for section, state in (
+            ("cohort", self._cohort_state),
+            ("frontier", self._frontier_state),
+        ):
+            if not state:
+                continue
+            stored = base.get(section)
+            previous: dict[str, Any] = stored if isinstance(stored, dict) else {}
+            if section == "frontier" and self._config_changed:
+                previous = {}
+            merged[section] = {**previous, **state}
+        return merged
 
     @property
     def cohort_stats(self) -> dict[str, Any]:

{classifyre_cli-0.6.7 → classifyre_cli-0.6.9}/src/outputs/rest.py

@@ -186,6 +186,9 @@ class FinalizeIngestRunRequest(BaseModel):
     # AUTOMATIC sampling cursor to persist on the source for the next run.
     # Omitted (None) for other strategies so the stored cursor is left untouched.
     sampling_cursor: dict[str, Any] | None = Field(None, serialization_alias="samplingCursor")
+    # The settings fingerprint the run was started with, echoed back so the API
+    # can record that those settings have now been scanned under.
+    sampling_fingerprint: str | None = Field(None, serialization_alias="samplingFingerprint")
     # What the scan cache saved, so the run can report it.
     assets_skipped_cached: int | None = Field(None, serialization_alias="assetsSkippedCached")
     detector_runs_skipped: int | None = Field(None, serialization_alias="detectorRunsSkipped")
@@ -248,6 +251,7 @@ class RestOutputSink:
         self._runner_id = context.runner_id
         self._seen_hashes: set[str] = set()
         self._sampling_cursor: dict[str, Any] | None = None
+        self._sampling_fingerprint: str | None = None
         self._scan_cache_savings: tuple[int, int] | None = None
         self._partial_coverage: bool = False
         self._partial_coverage_reason: str = ""
@@ -258,6 +262,10 @@ class RestOutputSink:
         """Record the AUTOMATIC sampling cursor to persist on finalize."""
         self._sampling_cursor = cursor
 
+    def set_sampling_fingerprint(self, fingerprint: str | None) -> None:
+        """Record the settings fingerprint this run was started with."""
+        self._sampling_fingerprint = fingerprint
+
     def set_partial_coverage(self, reason: str = "") -> None:
         """Mark this run as having covered only part of the source."""
         self._partial_coverage = True
@@ -364,6 +372,7 @@ class RestOutputSink:
             runner_id=runner_id,
             seen_hashes=sorted(self._seen_hashes),
             sampling_cursor=self._sampling_cursor,
+            sampling_fingerprint=self._sampling_fingerprint,
             assets_skipped_cached=savings[0] if savings else None,
             detector_runs_skipped=savings[1] if savings else None,
             # Sent even when zero: "this run emitted edges and lost none" is a