classifyre-cli 0.4.71__tar.gz → 0.4.73__tar.gz

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 302 packages in 216ms
+Resolved 303 packages in 168ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.71
+Version: 0.4.73
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.71",
+  "version": "0.4.73",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.71"
+version = "0.4.73"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -230,6 +230,9 @@ google-cloud-storage = [
 dropbox = [
     "dropbox>=12.2.1,<13.0.0",
 ]
+slack = [
+    "slack-sdk>=3.43.0,<4.0.0",
+]
 email = [
     "imap-tools>=1.10.0,<2.0.0",
 ]

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/src/detectors/pii/detector.py

@@ -107,7 +107,7 @@ class PIIDetector(BaseDetector):
         # Italy
         "IT_FISCAL_CODE",
         "IT_DRIVER_LICENSE",
-        "IT_VAR_CODE",
+        "IT_VAT_CODE",
         "IT_PASSPORT",
         "IT_IDENTITY_CARD",
         # Singapore
@@ -314,6 +314,7 @@ class PIIDetector(BaseDetector):
             else:
                 self.analyzer = AnalyzerEngine()
 
+            self._register_regional_recognizers(presidio_module)
             self._register_custom_recognizers(presidio_module)
             self._probe_phone_recognizer()
 
@@ -396,6 +397,129 @@ class PIIDetector(BaseDetector):
         logger.debug("Loaded spaCy model '%s'", cfg_model)
         return nlp_engine
 
+    # Regional recognizers Presidio ships but scopes to their own language, so an
+    # `supported_languages=["en"]` registry silently drops them (the "Recognizer
+    # not added to registry" warning is filtered out by _PresidioNoiseFilter).
+    # Every entity advertised in _ALL_SUPPORTED_ENTITIES has to be re-registered
+    # against "en" or it can never produce a finding, no matter what the preset
+    # asks for. Analysis always runs with language="en".
+    _REGIONAL_RECOGNIZER_CLASS_NAMES: ClassVar[tuple[str, ...]] = (
+        # Spain
+        "EsNifRecognizer",
+        "EsNieRecognizer",
+        # Italy
+        "ItFiscalCodeRecognizer",
+        "ItDriverLicenseRecognizer",
+        "ItVatCodeRecognizer",
+        "ItPassportRecognizer",
+        "ItIdentityCardRecognizer",
+        # Singapore
+        "SgFinRecognizer",
+        "SgUenRecognizer",
+        # Australia
+        "AuAbnRecognizer",
+        "AuAcnRecognizer",
+        "AuTfnRecognizer",
+        "AuMedicareRecognizer",
+        # India
+        "InPanRecognizer",
+        "InAadhaarRecognizer",
+        "InVehicleRegistrationRecognizer",
+        "InVoterRecognizer",
+        # Finland
+        "FiPersonalIdentityCodeRecognizer",
+        # Poland
+        "PlPeselRecognizer",
+    )
+
+    # DACH and EU identifiers Presidio has no recognizer for at all. Each pattern
+    # is deliberately anchored on a checkable structure and paired with context
+    # words, because a bare run of digits is far too common to flag on its own.
+    #
+    # Scores are chosen against Presidio's LemmaContextAwareEnhancer, which adds
+    # 0.35 when a context word sits near the match, and the detector's default
+    # confidence_threshold of 0.7. A structurally weak pattern therefore scores
+    # 0.4: on its own it stays below the threshold and is suppressed, and only a
+    # nearby context word lifts it to 0.75 and reports it. Anything lower could
+    # never surface at all; anything higher would fire on bare digit runs.
+    _DACH_PATTERN_SPECS: ClassVar[
+        tuple[tuple[str, str, tuple[tuple[str, str, float], ...], tuple[str, ...]], ...]
+    ] = (
+        (
+            "AT_SVNR",
+            "Austrian social insurance number (SVNR)",
+            # NNNN DDMMYY — a 4-digit serial (last digit is a check digit)
+            # followed by the holder's date of birth.
+            (("svnr (needs context)", r"\b\d{4}[ /]?\d{2}[01]\d{3}\b", 0.4),),
+            ("svnr", "sozialversicherung", "sozialversicherungsnummer", "versicherungsnummer"),
+        ),
+        (
+            "CH_AHV",
+            "Swiss social security number (AHV/AVS)",
+            # Always begins with the 756 country prefix, so this one is safe to
+            # score highly on structure alone.
+            (("ahv (strong)", r"\b756[.\s]?\d{4}[.\s]?\d{4}[.\s]?\d{2}\b", 0.7),),
+            ("ahv", "avs", "ahv-nummer", "sozialversicherung", "versichertennummer"),
+        ),
+        (
+            "DE_TAX_ID",
+            "German tax identification number (Steuer-IdNr)",
+            # 11 digits, never leading zero.
+            (("steuer-idnr (needs context)", r"\b[1-9]\d{2}[ /]?\d{4}[ /]?\d{4}\b", 0.4),),
+            ("steuer", "steuernummer", "steuer-idnr", "steueridentifikationsnummer", "tax id"),
+        ),
+        (
+            "EU_NATIONAL_ID",
+            "Generic EU national identity number",
+            (("eu national id (needs context)", r"\b[A-Z]{2}[ -]?\d{6,12}\b", 0.4),),
+            ("national id", "identity card", "personalausweis", "ausweisnummer", "id number"),
+        ),
+    )
+
+    def _register_regional_recognizers(self, presidio_module: Any) -> None:
+        """Re-register language-scoped and missing regional recognizers under "en"."""
+        if self.analyzer is None:
+            return
+
+        predefined = importlib.import_module("presidio_analyzer.predefined_recognizers")
+        already_supported = set(self.analyzer.get_supported_entities())
+
+        for class_name in self._REGIONAL_RECOGNIZER_CLASS_NAMES:
+            recognizer_class = getattr(predefined, class_name, None)
+            if recognizer_class is None:
+                logger.debug("Presidio recognizer %s not available; skipping", class_name)
+                continue
+            try:
+                recognizer = recognizer_class(supported_language="en")
+            except Exception as exc:
+                logger.debug("Could not instantiate %s: %s", class_name, exc)
+                continue
+            # Never double-register: a duplicate recognizer reports every match twice.
+            if already_supported.issuperset(recognizer.supported_entities):
+                continue
+            self.analyzer.registry.add_recognizer(recognizer)
+            already_supported.update(recognizer.supported_entities)
+
+        PatternRecognizer = presidio_module.PatternRecognizer  # noqa: N806
+        Pattern = presidio_module.Pattern  # noqa: N806
+
+        for entity, name, pattern_specs, context in self._DACH_PATTERN_SPECS:
+            if entity in already_supported:
+                continue
+            self.analyzer.registry.add_recognizer(
+                PatternRecognizer(
+                    supported_entity=entity,
+                    name=name,
+                    supported_language="en",
+                    patterns=[
+                        Pattern(name=pattern_name, regex=regex, score=score)
+                        for pattern_name, regex, score in pattern_specs
+                    ],
+                    context=list(context),
+                )
+            )
+            already_supported.add(entity)
+
     def _register_custom_recognizers(self, presidio_module: Any) -> None:
         """Add ad-hoc recognizers from config to the analyzer registry."""
         custom_recognizers = getattr(self.config, "custom_recognizers", None) or []
@@ -826,7 +950,7 @@ class PIIDetector(BaseDetector):
             "AU_ABN",
             "AU_ACN",
             "SG_UEN",
-            "IT_VAR_CODE",
+            "IT_VAT_CODE",
             "IN_VOTER",
             "IN_VEHICLE_REGISTRATION",
         }:

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/src/models/generated_detectors.py

@@ -258,7 +258,7 @@ class SecretsEnabledPattern(StrEnum):
 
 class PIIEnabledPattern(StrEnum):
     """
-    Presidio entity types for PII detection. Global: CREDIT_CARD, CRYPTO, DATE_TIME, EMAIL_ADDRESS, IBAN_CODE, IP_ADDRESS, NRP, LOCATION, PERSON, PHONE_NUMBER, MEDICAL_LICENSE, URL. USA: US_BANK_NUMBER, US_DRIVER_LICENSE, US_ITIN, US_PASSPORT, US_SSN. UK: UK_NHS. Spain: ES_NIF, ES_NIE. Italy: IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_VAR_CODE, IT_PASSPORT, IT_IDENTITY_CARD. Singapore: SG_NRIC_FIN, SG_UEN. Australia: AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE. India: IN_PAN, IN_AADHAAR, IN_VEHICLE_REGISTRATION, IN_VOTER. Finland: FI_PERSONAL_IDENTITY_CODE. Poland: PL_PESEL. DACH: AT_SVNR, CH_AHV, DE_TAX_ID, EU_NATIONAL_ID.
+    Presidio entity types for PII detection. Global: CREDIT_CARD, CRYPTO, DATE_TIME, EMAIL_ADDRESS, IBAN_CODE, IP_ADDRESS, NRP, LOCATION, PERSON, PHONE_NUMBER, MEDICAL_LICENSE, URL. USA: US_BANK_NUMBER, US_DRIVER_LICENSE, US_ITIN, US_PASSPORT, US_SSN. UK: UK_NHS. Spain: ES_NIF, ES_NIE. Italy: IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_VAT_CODE, IT_PASSPORT, IT_IDENTITY_CARD. Singapore: SG_NRIC_FIN, SG_UEN. Australia: AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE. India: IN_PAN, IN_AADHAAR, IN_VEHICLE_REGISTRATION, IN_VOTER. Finland: FI_PERSONAL_IDENTITY_CODE. Poland: PL_PESEL. DACH: AT_SVNR, CH_AHV, DE_TAX_ID, EU_NATIONAL_ID.
     """
 
     CREDIT_CARD = 'CREDIT_CARD'
@@ -283,7 +283,7 @@ class PIIEnabledPattern(StrEnum):
     ES_NIE = 'ES_NIE'
     IT_FISCAL_CODE = 'IT_FISCAL_CODE'
     IT_DRIVER_LICENSE = 'IT_DRIVER_LICENSE'
-    IT_VAR_CODE = 'IT_VAR_CODE'
+    IT_VAT_CODE = 'IT_VAT_CODE'
     IT_PASSPORT = 'IT_PASSPORT'
     IT_IDENTITY_CARD = 'IT_IDENTITY_CARD'
     SG_NRIC_FIN = 'SG_NRIC_FIN'

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/src/models/generated_input.py

@@ -85,7 +85,7 @@ class PostStatus(StrEnum):
 
 class SlackChannelType(StrEnum):
     """
-    Slack conversation types to include
+    Slack conversation types to include. public_channel and private_channel are named channels, mpim is a group direct message, im is a one-to-one direct message.
     """
 
     public_channel = 'public_channel'
@@ -439,104 +439,91 @@ class SlackRequired(BaseModel):
         extra='forbid',
     )
     workspace: str | None = Field(
-        None, description='Slack workspace name or domain (for display and stable IDs)'
-    )
-
-
-class SlackMaskedBotToken(BaseModel):
-    model_config = ConfigDict(
-        extra='forbid',
-    )
-    bot_token: str = Field(..., description='Slack bot token (starts with xoxb-)')
-
-
-class SlackMaskedUserToken(BaseModel):
-    model_config = ConfigDict(
-        extra='forbid',
+        None,
+        description='Slack workspace name or domain, used for permalinks and stable asset IDs (example: acme or acme.slack.com)',
     )
-    user_token: str = Field(..., description='Slack user token (starts with xoxp-)')
 
 
-class TokenType(StrEnum):
+class SlackMasked(BaseModel):
     """
-    Token type hint
+    Bot-token authentication. Install the app into the workspace and copy its bot token; user tokens are not supported.
     """
 
-    bot = 'bot'
-    user = 'user'
-
-
-class SlackMaskedToken(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
-    token: str = Field(..., description='Slack token (bot or user)')
-    token_type: TokenType | None = Field(None, description='Token type hint')
+    bot_token: str = Field(
+        ...,
+        description='Slack bot token, starts with xoxb-. Needs the scopes channels:read, groups:read, im:read, mpim:read, channels:history, groups:history, im:history, mpim:history, users:read and files:read.',
+    )
 
 
 class SlackOptionalChannels(BaseModel):
     """
-    Channel discovery and targeting controls.
+    Which conversations to discover and scan.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
     channel_types: list[SlackChannelType] | None = Field(
-        ['public_channel'],
-        description='Slack conversation types to include when listing channels',
+        ['public_channel', 'private_channel', 'mpim', 'im'],
+        description='Conversation types to include when listing channels. The bot only sees private channels and group DMs it has been invited to.',
     )
     channel_ids: list[str] | None = Field(
         None,
-        description='Explicit channel IDs to scan. If provided, channel_types is ignored.',
+        description='Explicit conversation IDs to scan (example: C0123456789). When set, channel_types is ignored.',
     )
     exclude_archived: bool | None = Field(
-        True, description='Exclude archived channels when listing'
+        True, description='Skip archived channels when listing'
+    )
+    auto_join_public_channels: bool | None = Field(
+        False,
+        description='Join public channels the bot is not a member of before reading them. conversations.history returns not_in_channel otherwise. Requires the channels:join scope.',
     )
 
 
 class SlackOptionalTimeRange(BaseModel):
     """
-    Time window filters for message ingestion.
+    Limit ingestion to messages inside a time window.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
     oldest: str | None = Field(
-        None, description='Start of date range (Slack timestamp or ISO 8601)'
+        None,
+        description='Start of the range, as a Slack timestamp or an ISO 8601 datetime',
     )
     latest: str | None = Field(
-        None, description='End of date range (Slack timestamp or ISO 8601)'
+        None,
+        description='End of the range, as a Slack timestamp or an ISO 8601 datetime',
     )
 
 
 class SlackOptionalIngestion(BaseModel):
     """
-    Throughput and payload controls for Slack ingestion.
+    Throughput and message-content controls.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
     batch_size: int | None = Field(
-        200, description='Messages per API call (max 200)', ge=1, le=200
+        200,
+        description='Messages requested per API call (Slack caps this at 200)',
+        ge=1,
+        le=200,
     )
     rate_limit_delay_seconds: float | None = Field(
-        1, description='Delay between API calls to avoid rate limits', ge=0.0
+        1,
+        description="Extra pause between API calls. Slack's 429 responses are retried automatically; this only smooths out sustained traffic.",
+        ge=0.0,
     )
     include_thread_replies: bool | None = Field(
-        False, description='Include thread replies in fetched content for detectors'
-    )
-
-
-class SlackOptional(BaseModel):
-    model_config = ConfigDict(
-        extra='forbid',
+        True,
+        description="Fetch each thread's replies and scan them alongside the parent message. Costs one extra API call per threaded message.",
     )
-    channels: SlackOptionalChannels | None = None
-    time_range: SlackOptionalTimeRange | None = None
-    ingestion: SlackOptionalIngestion | None = None
 
 
 class ObjectStorageOptionalScope(BaseModel):
@@ -2130,30 +2117,6 @@ class CoreInput(BaseModel):
     )
 
 
-class SlackInput(CoreInput):
-    type: Literal['SLACK'] | None = Field(
-        None, description='Type of the asset or source'
-    )
-    required: SlackRequired
-    masked: SlackMaskedBotToken | SlackMaskedUserToken | SlackMaskedToken = Field(
-        ..., title='SlackMasked'
-    )
-    optional: SlackOptional | None = None
-    detectors: list[Detector] | None = Field(
-        None, description='Detectors to run on ingested content'
-    )
-    custom_detectors: list[CustomDetectorSelection] | None = Field(
-        None,
-        description='Reusable custom detector IDs selected from the custom detector catalog.',
-    )
-    sampling: SamplingConfig
-    resources: ResourceOverrides | None = None
-    cleanup_removed_detector_findings: bool | None = Field(
-        True,
-        description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
-    )
-
-
 class EmailInput(CoreInput):
     type: Literal['EMAIL'] | None = Field(
         None, description='Type of the asset or source'
@@ -4437,6 +4400,32 @@ class DropboxInput(CoreInput):
     )
 
 
+class SlackOptionalAttachments(BaseModel):
+    """
+    Files shared in messages. Each file becomes its own asset, and its text is extracted with the same parser used for local files (PDF, Office, images via OCR, audio and video transcription).
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_attachments: bool | None = Field(
+        True, description='Emit files shared in messages as scannable assets'
+    )
+    max_attachment_bytes: int | None = Field(
+        26214400,
+        description='Skip files larger than this many bytes (default 25 MB)',
+        ge=1024,
+        le=524288000,
+    )
+    include_file_extensions: list[str] | None = Field(
+        None,
+        description='Only download files with these extensions (example: .pdf, .docx). Empty means all extensions.',
+    )
+    exclude_file_extensions: list[str] | None = Field(
+        None, description='Skip files with these extensions (example: .mp4, .zip)'
+    )
+
+
 class YouTubeInput(CoreInput):
     type: Literal['YOUTUBE'] | None = Field(
         None, description='Type of the asset or source'
@@ -4459,6 +4448,38 @@ class YouTubeInput(CoreInput):
     )
 
 
+class SlackOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    channels: SlackOptionalChannels | None = None
+    time_range: SlackOptionalTimeRange | None = None
+    ingestion: SlackOptionalIngestion | None = None
+    attachments: SlackOptionalAttachments | None = None
+
+
+class SlackInput(CoreInput):
+    type: Literal['SLACK'] | None = Field(
+        None, description='Type of the asset or source'
+    )
+    required: SlackRequired
+    masked: SlackMasked
+    optional: SlackOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+    cleanup_removed_detector_findings: bool | None = Field(
+        True,
+        description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
+    )
+
+
 class DeltaLakeOptional(BaseModel):
     model_config = ConfigDict(
         extra='forbid',

{classifyre_cli-0.4.71 → classifyre_cli-0.4.73}/src/sources/notion/source.py

@@ -228,6 +228,12 @@ class NotionSource(BaseSource):
         self._external_file_url_by_hash: dict[str, str] = {}
         self._notion_signed_url_by_hash: dict[str, str] = {}
         self._file_block_id_by_hash: dict[str, str] = {}
+        # Asset IDs whose bytes have already been downloaded and put through the
+        # full extraction pipeline. Read by ParsedContentProvider and
+        # DetectorPipeline to suppress their fallback path — without it every
+        # asset that yields no text (an icon, a cover, a diagram) is downloaded
+        # and OCR'd a second time.
+        self._content_pages_processed: set[str] = set()
 
     # ------------------------------------------------------------------ config
     def _optional(self) -> NotionOptional:
@@ -297,6 +303,7 @@ class NotionSource(BaseSource):
         self._notion_signed_url_by_hash = {}
         self._file_block_id_by_hash = {}
         self._attachment_name_by_hash = {}
+        self._content_pages_processed = set()
 
     # --------------------------------------------------------------- discovery
     def _discover_refs(self) -> list[dict[str, Any]]:
@@ -826,6 +833,10 @@ class NotionSource(BaseSource):
             declared_mime_type=mime,
             file_name=self._attachment_file_name(asset_id, asset_id),
         )
+        # Record the asset as processed even when extraction produced nothing:
+        # the download and the extraction pass both already happened, and a
+        # second identical attempt costs exactly as much to return nothing again.
+        self._content_pages_processed.add(asset_id)
         if parsed.text_content:
             self._asset_content_cache[asset_id] = (parsed.raw_content, parsed.text_content)
             return parsed.raw_content, parsed.text_content