classifyre-cli 0.4.18__tar.gz → 0.4.20__tar.gz

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 256 packages in 167ms
+Resolved 262 packages in 282ms
 Checked 50 packages in 1ms

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.18
+Version: 0.4.20
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.18",
+  "version": "0.4.20",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.18"
+version = "0.4.20"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -125,6 +125,7 @@ detectors = [
     { include-group = "custom" },
     { include-group = "regex" },
     { include-group = "llm" },
+    { include-group = "transcription" },
 ]
 file-processing = [
     "filetype>=1.2.0",
@@ -134,6 +135,11 @@ file-processing = [
     "chardet>=7.4.3",
     "pyarrow>=18.0.0",
 ]
+transcription = [
+    # CPU audio/video transcription. faster-whisper decodes media via bundled
+    # PyAV (no system ffmpeg required) and runs ctranslate2 for inference.
+    "faster-whisper>=1.1.0",
+]
 ocr = [
     { include-group = "file-processing" },
     "docling>=2.94.0",
@@ -192,6 +198,10 @@ google-cloud-storage = [
 email = [
     "imap-tools>=1.10.0,<2.0.0",
 ]
+youtube = [
+    "yt-dlp>=2025.1.0",
+    "youtube-transcript-api>=1.0.0",
+]
 otel = [
     "opentelemetry-sdk>=1.42.0",
     "opentelemetry-exporter-otlp-proto-http>=1.27.0",
@@ -309,6 +319,8 @@ module = [
     "tldextract.*",
     "re2",
     "re2.*",
+    "faster_whisper",
+    "faster_whisper.*",
 ]
 ignore_missing_imports = true
 

classifyre_cli-0.4.20/src/config.py

@@ -0,0 +1,76 @@
+"""Runtime configuration loaded from environment variables.
+
+A central, reusable place for tunables that may be overridden via `.env`
+(loaded by ``src.main.load_local_env``) without touching source recipes. Each
+concrete config section is a small pydantic model with a cached accessor so the
+environment is read once per process.
+
+Future sources/processors can add their own sections here following the same
+pattern (``BaseModel`` + ``functools.lru_cache`` accessor).
+"""
+
+from __future__ import annotations
+
+import os
+from functools import lru_cache
+
+from pydantic import BaseModel, Field
+
+
+def _env_str(name: str, default: str) -> str:
+    value = os.environ.get(name)
+    if value is None:
+        return default
+    value = value.strip()
+    return value or default
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None or not raw.strip():
+        return default
+    try:
+        return int(raw.strip())
+    except ValueError:
+        return default
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None or not raw.strip():
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+class WhisperConfig(BaseModel):
+    """faster-whisper transcription settings (CPU-only defaults).
+
+    Overridable via environment so a deployment can trade speed for accuracy
+    (e.g. a larger model, GPU device, or float16 compute) without code changes.
+    """
+
+    model: str = Field(
+        "medium", description="Whisper model size or path (e.g. tiny, base, medium, large-v3)."
+    )
+    device: str = Field("cpu", description="Inference device: cpu, cuda, or auto.")
+    compute_type: str = Field(
+        "int8", description="ctranslate2 compute type: int8, int8_float16, float16, float32."
+    )
+    beam_size: int = Field(5, ge=1, description="Beam search width.")
+    vad_filter: bool = Field(
+        True, description="Drop non-speech segments with Silero VAD before decoding."
+    )
+    word_timestamps: bool = Field(True, description="Emit per-word timestamps during decoding.")
+
+
+@lru_cache(maxsize=1)
+def get_whisper_config() -> WhisperConfig:
+    """Return the process-wide WhisperConfig, populated from the environment."""
+    return WhisperConfig(
+        model=_env_str("CLASSIFYRE_WHISPER_MODEL", "medium"),
+        device=_env_str("CLASSIFYRE_WHISPER_DEVICE", "cpu"),
+        compute_type=_env_str("CLASSIFYRE_WHISPER_COMPUTE_TYPE", "int8"),
+        beam_size=_env_int("CLASSIFYRE_WHISPER_BEAM_SIZE", 5),
+        vad_filter=_env_bool("CLASSIFYRE_WHISPER_VAD_FILTER", True),
+        word_timestamps=_env_bool("CLASSIFYRE_WHISPER_WORD_TIMESTAMPS", True),
+    )

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/src/models/generated_input.py

@@ -44,15 +44,7 @@ class AssetType(StrEnum):
     SQLITE = 'SQLITE'
     NOTION = 'NOTION'
     EMAIL = 'EMAIL'
-
-
-class SourceCategory(StrEnum):
-    """
-    Category of the source: TABULAR for structured databases (PostgreSQL, MySQL, MSSQL, Oracle, Hive, Databricks Unity Catalog, Snowflake), UNSTRUCTURED for text/web/document sources (WordPress, S3-Compatible Storage, Azure Blob Storage, Google Cloud Storage, Slack, MongoDB, PowerBI, Tableau, Confluence, Jira, Service Desk)
-    """
-
-    TABULAR = 'TABULAR'
-    UNSTRUCTURED = 'UNSTRUCTURED'
+    YOUTUBE = 'YOUTUBE'
 
 
 class DetectorType(StrEnum):
@@ -114,6 +106,10 @@ class SamplingConfig(BaseModel):
         False,
         description='When true, enable OCR/text extraction for supported binary documents and images before routing text-capable detectors.',
     )
+    enable_transcription: bool | None = Field(
+        False,
+        description='When true, transcribe audio and video files to text (via faster-whisper) before routing text-capable detectors. Slower and requires the transcription dependency.',
+    )
     order_by_column: str | None = Field(
         None,
         description='Column to use for LATEST sampling mode in tabular sources (usually created_at/updated_at). Auto-detected when not set.',
@@ -318,6 +314,115 @@ class EmailOptional(BaseModel):
     scope: EmailOptionalScope | None = None
 
 
+class Type(StrEnum):
+    """
+    Type of the asset or source
+    """
+
+    WORDPRESS = 'WORDPRESS'
+    SLACK = 'SLACK'
+    S3_COMPATIBLE_STORAGE = 'S3_COMPATIBLE_STORAGE'
+    AZURE_BLOB_STORAGE = 'AZURE_BLOB_STORAGE'
+    GOOGLE_CLOUD_STORAGE = 'GOOGLE_CLOUD_STORAGE'
+    POSTGRESQL = 'POSTGRESQL'
+    MYSQL = 'MYSQL'
+    MSSQL = 'MSSQL'
+    ORACLE = 'ORACLE'
+    HIVE = 'HIVE'
+    DATABRICKS = 'DATABRICKS'
+    SNOWFLAKE = 'SNOWFLAKE'
+    MONGODB = 'MONGODB'
+    NEO4J = 'NEO4J'
+    POWERBI = 'POWERBI'
+    TABLEAU = 'TABLEAU'
+    CONFLUENCE = 'CONFLUENCE'
+    JIRA = 'JIRA'
+    SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
+    NOTION = 'NOTION'
+    EMAIL = 'EMAIL'
+    YOUTUBE = 'YOUTUBE'
+
+
+class YouTubeRequired(BaseModel):
+    """
+    Provide at least one of channels or video_urls (enforced at runtime).
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    channels: list[str] | None = Field(
+        None,
+        description='Channel URLs or handles to list videos from (e.g. https://www.youtube.com/@OpenAI or @OpenAI). At least one of channels/video_urls is required.',
+    )
+    video_urls: list[str] | None = Field(
+        None,
+        description='Explicit video watch URLs to scan (e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ). At least one of channels/video_urls is required.',
+    )
+
+
+class YouTubeMasked(BaseModel):
+    """
+    Optional credentials. Leave empty for public videos.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    cookies: str | None = Field(
+        None,
+        description='Netscape-format cookie file contents, used by yt-dlp to access age-restricted or members-only videos.',
+    )
+
+
+class YouTubeOptionalTranscript(BaseModel):
+    """
+    Transcript/caption fetching controls.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    languages: list[str] | None = Field(
+        None,
+        description='Preferred caption language codes in priority order (e.g. ["en"]). Empty means accept any available language.',
+    )
+    skip_transcript: bool | None = Field(
+        False,
+        description='When true, skip transcript fetching entirely (metadata-only assets, no detector content).',
+    )
+
+
+class YouTubeOptionalConnection(BaseModel):
+    """
+    Network controls for yt-dlp and transcript fetching.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    proxy_url: str | None = Field(
+        None,
+        description='Optional HTTP/HTTPS/SOCKS proxy URL to mitigate rate-limiting when scanning at scale.',
+    )
+    request_timeout_seconds: int | None = Field(
+        30, description='Socket timeout for yt-dlp network operations.', ge=1, le=300
+    )
+    ignore_errors: bool | None = Field(
+        True,
+        description='Continue past individual videos that fail to extract instead of aborting the run.',
+    )
+
+
+class YouTubeOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    transcript: YouTubeOptionalTranscript | None = None
+    connection: YouTubeOptionalConnection | None = None
+
+
 class SlackRequired(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
@@ -1926,35 +2031,6 @@ class CoreInput(BaseModel):
     resources: ResourceOverrides | None = None
 
 
-class Type(StrEnum):
-    """
-    Type of the asset or source
-    """
-
-    WORDPRESS = 'WORDPRESS'
-    SLACK = 'SLACK'
-    S3_COMPATIBLE_STORAGE = 'S3_COMPATIBLE_STORAGE'
-    AZURE_BLOB_STORAGE = 'AZURE_BLOB_STORAGE'
-    GOOGLE_CLOUD_STORAGE = 'GOOGLE_CLOUD_STORAGE'
-    POSTGRESQL = 'POSTGRESQL'
-    MYSQL = 'MYSQL'
-    MSSQL = 'MSSQL'
-    ORACLE = 'ORACLE'
-    HIVE = 'HIVE'
-    DATABRICKS = 'DATABRICKS'
-    SNOWFLAKE = 'SNOWFLAKE'
-    MONGODB = 'MONGODB'
-    NEO4J = 'NEO4J'
-    POWERBI = 'POWERBI'
-    TABLEAU = 'TABLEAU'
-    CONFLUENCE = 'CONFLUENCE'
-    JIRA = 'JIRA'
-    SERVICEDESK = 'SERVICEDESK'
-    SQLITE = 'SQLITE'
-    NOTION = 'NOTION'
-    EMAIL = 'EMAIL'
-
-
 class SlackInput(CoreInput):
     type: Literal['SLACK'] = Field('SLACK', description='Type of the asset or source')
     required: SlackRequired
@@ -2428,7 +2504,7 @@ class ConfluenceOptionalConnection(BaseModel):
     )
 
 
-class Type17(StrEnum):
+class Type18(StrEnum):
     """
     Filter spaces by space type
     """
@@ -2465,7 +2541,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
     keys: list[str] | None = Field(
         None, description='Filter spaces by keys (up to 250)', max_length=250
     )
-    type: Type17 | None = Field(None, description='Filter spaces by space type')
+    type: Type18 | None = Field(None, description='Filter spaces by space type')
     status: Status | None = Field(None, description='Filter spaces by status')
     labels: list[str] | None = Field(
         None,
@@ -2731,7 +2807,7 @@ class ServiceDeskOptional(BaseModel):
     content: ServiceDeskOptionalContent | None = None
 
 
-class Type18(StrEnum):
+class Type19(StrEnum):
     """
     Type of the asset or source
     """
@@ -2758,6 +2834,7 @@ class Type18(StrEnum):
     SQLITE = 'SQLITE'
     NOTION = 'NOTION'
     EMAIL = 'EMAIL'
+    YOUTUBE = 'YOUTUBE'
 
 
 class ConfluenceInput(CoreInput):
@@ -2995,6 +3072,24 @@ class NotionInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class YouTubeInput(CoreInput):
+    type: Literal['YOUTUBE'] = Field(
+        'YOUTUBE', description='Type of the asset or source'
+    )
+    required: YouTubeRequired
+    masked: YouTubeMasked | None = None
+    optional: YouTubeOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class SourceInput(
     RootModel[
         SlackInput
@@ -3019,6 +3114,7 @@ class SourceInput(
         | SQLiteInput
         | NotionInput
         | EmailInput
+        | YouTubeInput
     ]
 ):
     root: (
@@ -3044,6 +3140,7 @@ class SourceInput(
         | SQLiteInput
         | NotionInput
         | EmailInput
+        | YouTubeInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/src/sources/base.py

@@ -214,6 +214,12 @@ class BaseSource(ABC):
         sampling = getattr(config, "sampling", None) if config is not None else None
         return bool(getattr(sampling, "enable_ocr", False))
 
+    def transcription_enabled(self) -> bool:
+        """Return whether sampling-level audio/video transcription is enabled."""
+        config = getattr(self, "config", None)
+        sampling = getattr(config, "sampling", None) if config is not None else None
+        return bool(getattr(sampling, "enable_transcription", False))
+
     def parse_asset_bytes(
         self,
         file_bytes: bytes,
@@ -228,6 +234,7 @@ class BaseSource(ABC):
             declared_mime_type=declared_mime_type,
             file_name=file_name,
             enable_ocr=self.ocr_enabled(),
+            enable_transcription=self.transcription_enabled(),
         )
 
     def iter_asset_pages(
@@ -248,6 +255,7 @@ class BaseSource(ABC):
             include_column_names,
             file_name=file_name,
             enable_ocr=self.ocr_enabled(),
+            enable_transcription=self.transcription_enabled(),
         )
 
     async def fetch_content_bytes(self, asset_id: str) -> tuple[bytes, str] | None:

{classifyre_cli-0.4.18 → classifyre_cli-0.4.20}/src/sources/object_storage/base.py

@@ -360,14 +360,16 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         )
         normalized_mime = mime_type.split(";", 1)[0].strip().lower()
 
-        # Non-extractable types (images, audio, video, opaque binary) carry no text.
-        # Everything else defers extraction to fetch_content_pages() so detectors
-        # receive content in configurable-sized pages instead of one monolithic blob.
-        is_non_extractable = normalized_mime.startswith(
-            ("image/", "audio/", "video/")
-        ) or normalized_mime in (
-            "application/octet-stream",
-            "application/zip",
+        # Non-extractable types (images, opaque binary) carry no text. Audio/video
+        # are extractable only when transcription is enabled — otherwise they are
+        # treated as opaque binary. Everything else defers extraction to
+        # fetch_content_pages() so detectors receive content in configurable-sized
+        # pages instead of one monolithic blob.
+        is_media = normalized_mime.startswith(("audio/", "video/"))
+        is_non_extractable = (
+            normalized_mime.startswith("image/")
+            or (is_media and not self.transcription_enabled())
+            or normalized_mime in ("application/octet-stream", "application/zip")
         )
 
         return ContentSnapshot(

classifyre_cli-0.4.20/src/sources/youtube/__init__.py

@@ -0,0 +1,3 @@
+from .source import YouTubeSource
+
+__all__ = ["YouTubeSource"]