classifyre-cli 0.4.70__tar.gz → 0.4.71__tar.gz

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 302 packages in 146ms
+Resolved 302 packages in 216ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.70
+Version: 0.4.71
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.70",
+  "version": "0.4.71",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.70"
+version = "0.4.71"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/src/main.py

@@ -10,6 +10,7 @@ from typing import Any, cast
 
 from .outputs import create_output_sink
 from .sources import get_source, list_available_sources
+from .sources.dropbox.auth import add_dropbox_auth_arguments, run_dropbox_auth_command
 from .utils.validation import validate_input, validate_test_connection
 
 logger = logging.getLogger(__name__)
@@ -586,7 +587,7 @@ def main() -> None:
     parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
     parser.add_argument(
         "command",
-        choices=["test", "extract", "discover", "evaluate-file", "train"],
+        choices=["test", "extract", "discover", "evaluate-file", "train", "dropbox-auth"],
         help="Command to run",
     )
     parser.add_argument(
@@ -667,6 +668,7 @@ def main() -> None:
         default=None,
         help="Max assets processed concurrently in Phase 2. Controls DB connection usage. Defaults to pool_workers*2 (env: CLASSIFYRE_MAX_CONCURRENT_ASSETS)",
     )
+    add_dropbox_auth_arguments(parser)
 
     args = parser.parse_args()
 
@@ -695,6 +697,9 @@ def main() -> None:
     if args.debug:
         logging.getLogger().setLevel(logging.DEBUG)
 
+    if args.command == "dropbox-auth":
+        sys.exit(run_dropbox_auth_command(args))
+
     if args.command == "evaluate-file":
         run_evaluate_file_command(args)
         return

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/src/models/generated_input.py

@@ -3461,27 +3461,31 @@ class SASLUsernamePassword1(BaseModel):
     )
     sasl_username: str = Field(..., description='SASL username')
     sasl_password: str = Field(..., description='SASL password')
-    ssl_ca: str | None = Field(
+    ca_certificate: str | None = Field(
         None,
-        description="PEM-encoded CA certificate used to verify the broker's TLS certificate. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
+        description="PEM-encoded CA certificate used to verify the broker's TLS certificate — the provider's ca.pem. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
     )
 
 
 class ClientCertificateMTLS1(BaseModel):
     """
-    mTLS credentials: the client certificate and its private key, plus the CA certificate when the broker uses a private CA.
+    mTLS credentials, named as the providers name them: Aiven's access key / access certificate / CA certificate, Confluent's client key / client certificate. Paste each PEM block whole, including its BEGIN and END lines.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
-    ssl_certfile: str = Field(..., description='PEM-encoded client access certificate')
-    ssl_keyfile: str = Field(
-        ..., description='PEM-encoded client access key (private key)'
+    access_key: str = Field(
+        ...,
+        description='PEM-encoded client private key — the block starting with -----BEGIN PRIVATE KEY----- (Aiven: service.key)',
     )
-    ssl_ca: str | None = Field(
+    access_certificate: str = Field(
+        ...,
+        description='PEM-encoded client certificate — the block starting with -----BEGIN CERTIFICATE----- (Aiven: service.cert)',
+    )
+    ca_certificate: str | None = Field(
         None,
-        description="PEM-encoded CA certificate used to verify the broker's TLS certificate. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
+        description="PEM-encoded CA certificate used to verify the broker's TLS certificate — the provider's ca.pem. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
     )
 
 
@@ -4221,39 +4225,43 @@ class GoogleWorkspaceInput(CoreInput):
     )
 
 
-class DropboxRequiredAccessToken(BaseModel):
+class DropboxOAuthRefreshTokenRecommended(BaseModel):
     """
-    Access token issued from the Dropbox App Console. Simplest to set up; short-lived tokens expire after 4 hours.
+    Recommended. An OAuth app key and secret plus a refresh token, which never expires — the client mints a fresh access token on every run. Obtain the refresh token once with `classifyre dropbox-auth --app-key <key> --app-secret <secret>`; it prints a URL you open in your own browser and exchanges the code it gives you.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
-    auth_method: Literal['access_token']
+    auth_method: Literal['oauth']
+    app_key: str = Field(
+        ..., description='Dropbox app key (App Console → Settings → App key)'
+    )
 
 
-class DropboxRequiredOAuth(BaseModel):
+class DropboxOAuthWithPKCENoAppSecret(BaseModel):
     """
-    OAuth 2.0 app credentials with a refresh token. Recommended for unattended scans — the client mints a fresh access token on every run.
+    Same as the OAuth refresh token flow but for apps that hold no secret: the authorization is proved with PKCE, so only the app key is needed to refresh. Obtain the refresh token with `classifyre dropbox-auth --app-key <key> --pkce`.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
-    auth_method: Literal['oauth']
+    auth_method: Literal['oauth_pkce']
     app_key: str = Field(
-        ...,
-        description='Dropbox app key (client ID) of the OAuth app that issued the refresh token',
+        ..., description='Dropbox app key (App Console → Settings → App key)'
     )
 
 
-class DropboxMaskedAccessToken(BaseModel):
+class DropboxAccessTokenShortLivedTestingOnly(BaseModel):
+    """
+    Testing only. The token generated by the App Console button is short-lived and stops working about four hours later with `invalid_access_token`, so scheduled scans will fail. Use one of the refresh-token methods for anything recurring.
+    """
+
     model_config = ConfigDict(
         extra='forbid',
     )
-    access_token: str = Field(
-        ..., description='Dropbox OAuth 2.0 access token (sl.… for short-lived tokens)'
-    )
+    auth_method: Literal['access_token']
 
 
 class DropboxMaskedOAuth(BaseModel):
@@ -4261,12 +4269,31 @@ class DropboxMaskedOAuth(BaseModel):
         extra='forbid',
     )
     app_secret: str = Field(
+        ..., description='Dropbox app secret (App Console → Settings → App secret)'
+    )
+    refresh_token: str = Field(
         ...,
-        description='Dropbox app secret of the OAuth app that issued the refresh token',
+        description='Long-lived refresh token issued with token_access_type=offline',
+    )
+
+
+class DropboxMaskedOAuthPkce(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
     )
     refresh_token: str = Field(
         ...,
-        description='Dropbox OAuth 2.0 refresh token (obtained with token_access_type=offline)',
+        description='Long-lived refresh token issued by the PKCE flow with token_access_type=offline',
+    )
+
+
+class DropboxMaskedAccessToken(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    access_token: str = Field(
+        ...,
+        description='Short-lived Dropbox access token (starts with sl.). Expires roughly four hours after it is generated.',
     )
 
 
@@ -4386,11 +4413,13 @@ class DropboxInput(CoreInput):
     type: Literal['DROPBOX'] | None = Field(
         None, description='Type of the asset or source'
     )
-    required: DropboxRequiredAccessToken | DropboxRequiredOAuth = Field(
-        ..., title='DropboxRequired'
-    )
-    masked: DropboxMaskedAccessToken | DropboxMaskedOAuth = Field(
-        ..., title='DropboxMasked'
+    required: (
+        DropboxOAuthRefreshTokenRecommended
+        | DropboxOAuthWithPKCENoAppSecret
+        | DropboxAccessTokenShortLivedTestingOnly
+    ) = Field(..., title='DropboxRequired')
+    masked: DropboxMaskedOAuth | DropboxMaskedOAuthPkce | DropboxMaskedAccessToken = (
+        Field(..., title='DropboxMasked')
     )
     optional: DropboxOptional | None = None
     detectors: list[Detector] | None = Field(

classifyre_cli-0.4.71/src/sources/dropbox/auth.py

@@ -0,0 +1,254 @@
+"""One-off Dropbox authorization helper.
+
+Dropbox only hands out long-lived credentials through an interactive OAuth
+authorization. A scan runner has no browser and no logged-in user, so the
+exchange has to happen once, up front, on a machine where a human is present.
+
+``DropboxOAuth2FlowNoRedirect`` is built exactly for that: it needs no redirect
+URI and no callback server. It prints a URL, the operator opens it in their own
+browser, Dropbox shows them a short authorization code, and the code is pasted
+back here and exchanged for a refresh token. Only that refresh token — which
+does not expire — is ever stored on a source.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Any
+
+from ..dependencies import require_module
+
+# Scopes each mode needs. Dropbox binds scopes to the refresh token at
+# authorization time, so anything missing here means a `missing_scope` error at
+# scan time and a fresh authorization to fix it.
+INDIVIDUAL_SCOPES = [
+    "account_info.read",
+    "files.metadata.read",
+    "files.content.read",
+    "sharing.read",
+]
+
+TEAM_SCOPES = [
+    *INDIVIDUAL_SCOPES,
+    "team_info.read",
+    "members.read",
+    "team_data.member",
+    "team_data.team_space",
+]
+
+
+def default_scopes(*, team: bool) -> list[str]:
+    return list(TEAM_SCOPES if team else INDIVIDUAL_SCOPES)
+
+
+def build_flow(
+    *,
+    app_key: str,
+    app_secret: str | None,
+    scopes: list[str],
+    use_pkce: bool,
+) -> Any:
+    """Create the no-redirect authorization flow.
+
+    ``token_access_type="offline"`` is what makes Dropbox return a refresh
+    token; without it the exchange yields only a four-hour access token.
+    """
+    oauth = require_module(
+        module_name="dropbox.oauth",
+        source_name="Dropbox",
+        uv_groups=["dropbox"],
+        detail="Dropbox authorization requires the official Dropbox SDK.",
+    )
+    return oauth.DropboxOAuth2FlowNoRedirect(
+        app_key,
+        consumer_secret=None if use_pkce else app_secret,
+        token_access_type="offline",
+        scope=scopes,
+        use_pkce=use_pkce,
+    )
+
+
+def verify_credentials(
+    *,
+    app_key: str,
+    app_secret: str | None,
+    refresh_token: str,
+    team: bool,
+) -> str:
+    """Prove the refresh token works before printing it as a recipe."""
+    dropbox_module = require_module(
+        module_name="dropbox",
+        source_name="Dropbox",
+        uv_groups=["dropbox"],
+        detail="Dropbox authorization requires the official Dropbox SDK.",
+    )
+    kwargs: dict[str, Any] = {
+        "oauth2_refresh_token": refresh_token,
+        "app_key": app_key,
+        "user_agent": "classifyre",
+    }
+    if app_secret:
+        kwargs["app_secret"] = app_secret
+
+    if team:
+        client = dropbox_module.DropboxTeam(**kwargs)
+        info = client.team_get_info()
+        return (
+            f"team '{getattr(info, 'name', '')}' ({getattr(info, 'num_licensed_users', '?')} seats)"
+        )
+
+    client = dropbox_module.Dropbox(**kwargs)
+    account = client.users_get_current_account()
+    display_name = getattr(getattr(account, "name", None), "display_name", "") or ""
+    return f"{display_name} <{getattr(account, 'email', '')}>"
+
+
+def build_recipe_credentials(
+    *,
+    app_key: str,
+    app_secret: str | None,
+    refresh_token: str,
+    use_pkce: bool,
+) -> dict[str, Any]:
+    """The `required`/`masked` blocks to paste into a DROPBOX source."""
+    if use_pkce:
+        return {
+            "required": {"auth_method": "oauth_pkce", "app_key": app_key},
+            "masked": {"refresh_token": refresh_token},
+        }
+    return {
+        "required": {"auth_method": "oauth", "app_key": app_key},
+        "masked": {"app_secret": app_secret or "", "refresh_token": refresh_token},
+    }
+
+
+def run_dropbox_auth_command(args: argparse.Namespace) -> int:
+    app_key = str(getattr(args, "app_key", "") or "").strip()
+    app_secret = str(getattr(args, "app_secret", "") or "").strip() or None
+    use_pkce = bool(getattr(args, "pkce", False))
+    team = bool(getattr(args, "team", False))
+
+    if not app_key:
+        print("--app-key is required (Dropbox App Console → Settings → App key)")
+        return 1
+    if not use_pkce and not app_secret:
+        print(
+            "--app-secret is required unless --pkce is used.\n"
+            "Use --pkce for an app that cannot hold a secret; it refreshes with the app key alone."
+        )
+        return 1
+
+    scopes = list(getattr(args, "scope", None) or []) or default_scopes(team=team)
+
+    try:
+        flow = build_flow(
+            app_key=app_key,
+            app_secret=app_secret,
+            scopes=scopes,
+            use_pkce=use_pkce,
+        )
+        authorize_url = flow.start()
+    except Exception as exc:
+        print(f"Failed to start the Dropbox authorization flow: {exc}")
+        return 1
+
+    print()
+    print("Dropbox authorization — this runs once; the runner never needs a browser.")
+    print(f"  Mode:   {'PKCE (no app secret)' if use_pkce else 'app key + secret'}")
+    print(f"  Target: {'Dropbox Business team' if team else 'individual account'}")
+    print(f"  Scopes: {' '.join(scopes)}")
+    print()
+    print("1. Open this URL in your browser:")
+    print(f"   {authorize_url}")
+    print('2. Sign in if needed, then click "Allow".')
+    print("3. Copy the authorization code Dropbox shows you.")
+    print()
+
+    try:
+        auth_code = input("Authorization code: ").strip()
+    except (EOFError, KeyboardInterrupt):
+        print("\nAborted.")
+        return 1
+
+    if not auth_code:
+        print("No authorization code entered.")
+        return 1
+
+    try:
+        result = flow.finish(auth_code)
+    except Exception as exc:
+        print(f"Failed to exchange the authorization code: {exc}")
+        return 1
+
+    refresh_token = getattr(result, "refresh_token", None)
+    if not refresh_token:
+        print(
+            "Dropbox returned no refresh token. The app must be authorized with "
+            "token_access_type=offline; re-run this command."
+        )
+        return 1
+
+    try:
+        identity = verify_credentials(
+            app_key=app_key,
+            app_secret=app_secret,
+            refresh_token=refresh_token,
+            team=team,
+        )
+    except Exception as exc:
+        print(f"Refresh token issued, but the verification call failed: {exc}")
+        if team:
+            print(
+                "For a team scan the Dropbox app must use a team access type "
+                "('Team member file access')."
+            )
+        return 1
+
+    credentials = build_recipe_credentials(
+        app_key=app_key,
+        app_secret=app_secret,
+        refresh_token=refresh_token,
+        use_pkce=use_pkce,
+    )
+
+    print()
+    print(f"Authorized as {identity}.")
+    print("Paste these into the Dropbox source configuration:")
+    print()
+    print(json.dumps(credentials, indent=2))
+    print()
+    print("The refresh token does not expire. Treat it as a secret.")
+    return 0
+
+
+def add_dropbox_auth_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--app-key", default=None, help="Dropbox app key (dropbox-auth only)")
+    parser.add_argument(
+        "--app-secret",
+        default=None,
+        help="Dropbox app secret; omit when using --pkce (dropbox-auth only)",
+    )
+    parser.add_argument(
+        "--pkce",
+        action="store_true",
+        help="Authorize with PKCE so no app secret is needed (dropbox-auth only)",
+    )
+    parser.add_argument(
+        "--team",
+        action="store_true",
+        help="Authorize a Dropbox Business team app and request team scopes (dropbox-auth only)",
+    )
+    parser.add_argument(
+        "--scope",
+        action="append",
+        default=None,
+        help="Override a requested OAuth scope; repeatable (dropbox-auth only)",
+    )
+
+
+def main() -> None:  # pragma: no cover - thin argparse wrapper
+    parser = argparse.ArgumentParser(description="Authorize Classifyre against Dropbox")
+    add_dropbox_auth_arguments(parser)
+    sys.exit(run_dropbox_auth_command(parser.parse_args()))

{classifyre_cli-0.4.70 → classifyre_cli-0.4.71}/src/sources/dropbox/source.py

@@ -7,7 +7,21 @@ from datetime import UTC, datetime
 from typing import Any
 from urllib.parse import quote
 
-from ...models.generated_input import DropboxInput, DropboxRequiredOAuth
+# The generated model names come from each auth branch's schema `title`, which
+# is also what the source form shows in its auth dropdown. Alias them back to
+# stable names so renaming a label never churns this module.
+from ...models.generated_input import (
+    DropboxAccessTokenShortLivedTestingOnly as DropboxRequiredAccessToken,
+)
+from ...models.generated_input import (
+    DropboxInput,
+)
+from ...models.generated_input import (
+    DropboxOAuthRefreshTokenRecommended as DropboxRequiredOAuth,
+)
+from ...models.generated_input import (
+    DropboxOAuthWithPKCENoAppSecret as DropboxRequiredOAuthPkce,
+)
 from ...models.generated_single_asset_scan_results import SingleAssetScanResults
 from ..dependencies import require_module
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
@@ -178,6 +192,11 @@ class DropboxSource(ObjectStorageSourceBase):
         Both client classes take the same credentials; what differs between an
         individual and a Business scan is the access type granted to the app,
         not the shape of the secret.
+
+        Refresh-token modes are the only ones that survive a scheduled run: the
+        SDK exchanges the refresh token for a new access token on every call
+        that needs one. PKCE apps refresh with the app key alone, so
+        ``app_secret`` is omitted rather than empty.
         """
         common: dict[str, Any] = {
             "timeout": self._request_timeout_seconds(),
@@ -187,27 +206,64 @@ class DropboxSource(ObjectStorageSourceBase):
         }
 
         required = self.config.required
-        if isinstance(required, DropboxRequiredOAuth):
+
+        if isinstance(required, DropboxRequiredOAuth | DropboxRequiredOAuthPkce):
             app_key = str(required.app_key or "").strip()
-            app_secret = self._masked_value("app_secret")
             refresh_token = self._masked_value("refresh_token")
-            if not app_key or not app_secret or not refresh_token:
+            if not app_key or not refresh_token:
                 raise ValueError(
-                    "Dropbox OAuth authentication requires required.app_key, "
-                    "masked.app_secret and masked.refresh_token"
+                    "Dropbox refresh-token authentication requires required.app_key "
+                    "and masked.refresh_token. Run "
+                    "`classifyre dropbox-auth --app-key <key>` to obtain one."
                 )
-            return {
+            credentials: dict[str, Any] = {
                 "oauth2_refresh_token": refresh_token,
                 "app_key": app_key,
-                "app_secret": app_secret,
                 **common,
             }
+            if isinstance(required, DropboxRequiredOAuth):
+                app_secret = self._masked_value("app_secret")
+                if not app_secret:
+                    raise ValueError(
+                        "Dropbox OAuth authentication requires masked.app_secret. "
+                        "Use auth_method 'oauth_pkce' for an app that holds no secret."
+                    )
+                credentials["app_secret"] = app_secret
+            return credentials
 
         access_token = self._masked_value("access_token")
         if not access_token:
             raise ValueError("Dropbox access token authentication requires masked.access_token")
         return {"oauth2_access_token": access_token, **common}
 
+    def _uses_short_lived_token(self) -> bool:
+        return isinstance(self.config.required, DropboxRequiredAccessToken)
+
+    def _explain_auth_failure(self, exc: Exception) -> str:
+        """Turn Dropbox's opaque auth errors into something actionable."""
+        text = str(exc)
+        if "invalid_access_token" in text and self._uses_short_lived_token():
+            return (
+                "the access token is no longer valid. App Console tokens are "
+                "short-lived and stop working a few hours after they are generated. "
+                "Switch this source to auth_method 'oauth' (or 'oauth_pkce') and run "
+                "`classifyre dropbox-auth --app-key <key> --app-secret <secret>` to "
+                "get a refresh token that does not expire."
+            )
+        if "invalid_access_token" in text:
+            return (
+                "the refresh token was rejected. It may have been revoked, or it may "
+                "belong to a different Dropbox app than the configured app key. "
+                "Re-run `classifyre dropbox-auth` to issue a new one."
+            )
+        if "missing_scope" in text:
+            return (
+                f"{text} — the refresh token was issued without a scope this scan needs. "
+                "Re-run `classifyre dropbox-auth` with the required --scope values so the "
+                "authorization covers them."
+            )
+        return text
+
     def _build_client(self) -> Any:
         return self._sdk().Dropbox(**self._credential_kwargs())
 
@@ -655,7 +711,7 @@ class DropboxSource(ObjectStorageSourceBase):
             )
         except Exception as exc:
             result["status"] = "FAILURE"
-            result["message"] = f"Failed to connect to Dropbox: {exc}"
+            result["message"] = f"Failed to connect to Dropbox: {self._explain_auth_failure(exc)}"
         return result
 
     async def extract_raw(self) -> AsyncGenerator[list[SingleAssetScanResults], None]: