pyetool 0.15.1__py3-none-any.whl

pyetool/tool/ai/__init__.py

@@ -0,0 +1,6 @@
+"""pyetool AI tools: LLM-backed utilities (currently: video visual analysis)."""
+
+from .cli import cli
+from .visual import Scene, VisualAnalyser, Visuals
+
+__all__ = ["Scene", "VisualAnalyser", "Visuals", "cli"]

pyetool/tool/ai/cli.py

@@ -0,0 +1,29 @@
+"""Top-level AI CLI: dispatches to per-tool subcommands.
+
+Each AI tool under `pyetool/tool/ai/` contributes its own click command
+(e.g. `visual`) which is registered here.
+
+All subcommands emit JSONL on stdout (one record per line) so they compose
+cleanly with `jq` and other JSONL tools.
+"""
+
+import click
+
+from .visual import cli as visual_cli
+
+
+@click.group()
+def cli() -> None:
+    """pyetool AI tools — LLM-based analyses, pipeline-friendly via JSONL.
+
+    \b
+    Subcommands:
+      visual VIDEO   analyse a video's visual content with Gemini, emit Visuals JSONL.
+    """
+
+
+cli.add_command(visual_cli)
+
+
+if __name__ == "__main__":
+    cli()

pyetool/tool/ai/visual/__init__.py

@@ -0,0 +1,5 @@
+from .cli import cli
+from .model import Scene, Visuals
+from .visual import VisualAnalyser
+
+__all__ = ["Scene", "VisualAnalyser", "Visuals", "cli"]

pyetool/tool/ai/visual/cli.py

@@ -0,0 +1,82 @@
+"""CLI for `aix visual`: analyse a video's visual content with Gemini, emit JSONL."""
+
+import json
+
+import click
+
+from pyetool.util.cli_util import collect_records, fail
+from pyetool.util.monad import Err, Ok
+
+from .visual import VisualAnalyser
+
+
+@click.command("visual")
+@click.argument("video", required=False)
+@click.option(
+    "--model",
+    default="gemini-2.5-flash",
+    show_default=True,
+    help="Gemini model id (e.g. gemini-2.5-flash, gemini-2.5-pro).",
+)
+@click.option(
+    "--max-attempts",
+    type=int,
+    default=5,
+    show_default=True,
+    help="Total tries (initial + retries) for the generate call when rate-limited (429).",
+)
+@click.option(
+    "--api-key",
+    default=None,
+    help="Gemini API key. Falls back to $GEMINI_API_KEY in ~/.config/pyetool/.env or ./.env.",
+)
+@click.option(
+    "--input-key",
+    default="path",
+    show_default=True,
+    help="JSONL field name carrying the source video path or URL. Default 'path' "
+    "matches the pyetool batch convention (vidkit / pexels download / etc.).",
+)
+def cli(video, model, max_attempts, api_key, input_key):
+    """Analyse a video's visual content with a Gemini model.
+
+    \b
+    VIDEO is a local file path or an http(s) URL (e.g. a YouTube URL).
+    Pass as a positional argument and/or pipe records via stdin (one per
+    line: a plain path/URL string, or a JSON object whose `--input-key` field
+    is the source).
+
+    \b
+    Output: pass-through JSONL — each input record with the following
+    fields appended:
+      summary  — overall 2-4 sentence description
+      scenes   — list of {summary, mood} per distinct shot, chronological
+      mood     — overall mood / tone
+
+    \b
+    Examples:
+      aix visual ./clip.mp4
+      aix visual 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+      printf './a.mp4\\n./b.mp4\\n' | aix visual --model gemini-2.5-pro
+      pexels videos search 'sunset' --per-page 3 \\
+        | pexels videos download ./pack/ \\
+        | vidkit batch-shrink ./shrunk/ \\
+        | aix visual
+    """
+    analyser = VisualAnalyser(model=model, api_key=api_key, max_attempts=max_attempts)
+
+    for record in collect_records(video, default_key=input_key, label="VIDEO"):
+        src = record.get(input_key)
+        if not src:
+            raise click.UsageError(f"record missing {input_key!r} field: {record!r}")
+
+        match analyser.analyse(src):
+            case Ok(value=visuals):
+                line = {**record, **visuals.model_dump(mode="json")}
+                click.echo(json.dumps(line, ensure_ascii=False))
+            case Err() as e:
+                fail(e)
+
+
+if __name__ == "__main__":
+    cli()

pyetool/tool/ai/visual/model.py

@@ -0,0 +1,18 @@
+"""Pydantic models for video visual analysis output."""
+
+from pydantic import BaseModel, Field
+
+
+class Scene(BaseModel):
+    """One distinct shot or scene in the video."""
+
+    summary: str = Field(description="What happens in this scene")
+    mood: str = Field(description="Mood or tone of this scene")
+
+
+class Visuals(BaseModel):
+    """Structured description of a video's visual content."""
+
+    summary: str = Field(description="2-4 sentence overall description of the video")
+    scenes: list[Scene] = Field(description="Distinct shots in chronological order")
+    mood: str = Field(description="Overall mood or tone of the video")

pyetool/tool/ai/visual/visual.py

@@ -0,0 +1,173 @@
+"""Gemini-based video visual analysis via pydantic-ai.
+
+Source: a local file path or an http(s) URL (e.g. YouTube). Local files are
+uploaded via the Gemini Files API and referenced by their generated URI;
+URLs are passed through to Gemini as-is.
+
+Output: `Visuals` — overall summary, per-scene breakdown, overall mood.
+
+Auth: pass `api_key` to `VisualAnalyser`, or set `GEMINI_API_KEY` in
+`~/.config/pyetool/.env` or `./.env`.
+
+Retries: on 429 RESOURCE_EXHAUSTED the server returns a `RetryInfo.retryDelay`
+hint inside the response body (not in the `Retry-After` header). The
+google-genai SDK's built-in retry ignores it; we extract it from
+`ModelHTTPError.body` surfaced by pydantic-ai and sleep for that duration.
+"""
+
+import os
+import time
+from pathlib import Path
+from typing import Any, ClassVar
+
+from google.genai.types import File
+from pydantic_ai import Agent, DocumentUrl, VideoUrl
+from pydantic_ai.exceptions import ModelHTTPError
+from pydantic_ai.models.google import GoogleModel
+from pydantic_ai.providers.google import GoogleProvider
+
+from pyetool.util.env import load_env
+from pyetool.util.monad import Err, Ok, Result
+
+from .model import Visuals
+
+
+class VisualAnalyser:
+    """Analyse a video's visual content with a Gemini model via pydantic-ai.
+
+    `analyse()` accepts a local path or an http(s) URL; local files are
+    uploaded via the Gemini Files API and then referenced by their URI.
+    """
+
+    _PROMPT: ClassVar[str] = "Describe the visual content of this video as structured data."
+    _BACKOFF_BASE: ClassVar[float] = 5.0
+    _POLL_INTERVAL: ClassVar[float] = 2.0
+    _UPLOAD_TIMEOUT: ClassVar[float] = 600.0
+
+    def __init__(
+        self,
+        model: str = "gemini-2.5-flash",
+        api_key: str | None = None,
+        max_attempts: int = 5,
+    ) -> None:
+        """Auth: pass `api_key`, or set `GEMINI_API_KEY` in `~/.config/pyetool/.env` / `./.env`.
+
+        `max_attempts` is the total number of tries (initial + retries) for
+        the generate call; the upload step is single-shot (Gemini rate limits
+        only the generation endpoint).
+        """
+        load_env()
+        if not api_key and not (api_key := os.environ.get("GEMINI_API_KEY")):
+            raise ValueError("api_key is not provided as parameter or GEMINI_API_KEY in .env")
+
+        self._provider = GoogleProvider(api_key=api_key)
+        self._agent = Agent(
+            GoogleModel(model, provider=self._provider),
+            output_type=Visuals,
+        )
+        self._max_attempts = max_attempts
+
+    def analyse(self, video: str | Path) -> Result[Visuals]:
+        """Analyse a local file or an http(s) URL.
+
+        Returns `Err` on missing file, upload failure, retries exhausted, or
+        any other non-recoverable API error.
+        """
+        try:
+            content = self._prepare_content(video)
+        except (FileNotFoundError, RuntimeError) as e:
+            return Err(err=str(e))
+
+        try:
+            return Ok(self._generate(content))
+        except ModelHTTPError as e:
+            return Err(err=f"Gemini API error: {e.body or e}", returncode=e.status_code)
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _prepare_content(self, video: str | Path) -> list[Any]:
+        if isinstance(video, str) and video.startswith(("http://", "https://")):
+            return [self._PROMPT, VideoUrl(url=video)]
+
+        path = Path(video)
+        if not path.exists():
+            raise FileNotFoundError(f"Video file not found: {path}")
+        file = self._upload(path)
+        assert file.uri is not None
+        assert file.mime_type is not None
+        return [self._PROMPT, DocumentUrl(url=file.uri, media_type=file.mime_type)]
+
+    def _upload(self, path: Path) -> File:
+        """Upload then poll until ACTIVE. Raises RuntimeError on FAILED / timeout."""
+        file = self._provider.client.files.upload(file=path)
+        deadline = time.monotonic() + self._UPLOAD_TIMEOUT
+
+        while file.state is not None and file.state.name == "PROCESSING":
+            if time.monotonic() >= deadline:
+                raise RuntimeError(f"Gemini file '{path.name}' stayed in PROCESSING for over {self._UPLOAD_TIMEOUT}s")
+            time.sleep(self._POLL_INTERVAL)
+            assert file.name is not None
+            file = self._provider.client.files.get(name=file.name)
+
+        if file.state is not None and file.state.name == "FAILED":
+            raise RuntimeError(
+                f"Gemini failed to process '{path.name}'; the file may be corrupted, "
+                f"in an unsupported format, or exceed size limits"
+            )
+        return file
+
+    def _generate(self, content: list[Any]) -> Visuals:
+        """Run the agent with retry on transient errors.
+
+        Retries on 429 RESOURCE_EXHAUSTED and any 5xx (UNAVAILABLE / INTERNAL
+        / etc.). For 429, the server's `RetryInfo.retryDelay` hint is honored
+        when present (5xx bodies normally do not carry it, so they fall back
+        to exponential backoff). Re-raises immediately on non-retryable 4xx.
+        """
+        last_exc: ModelHTTPError | None = None
+        for attempt in range(self._max_attempts):
+            try:
+                return self._agent.run_sync(content).output
+            except ModelHTTPError as e:
+                if e.status_code != 429 and e.status_code < 500:
+                    raise
+                last_exc = e
+                if attempt == self._max_attempts - 1:
+                    break
+                delay = self._parse_retry_delay(e.body) or self._BACKOFF_BASE * (2**attempt)
+                time.sleep(delay)
+
+        assert last_exc is not None
+        raise last_exc
+
+    @staticmethod
+    def _parse_retry_delay(body: object) -> float | None:
+        """Extract `retryDelay` seconds from a 429 body's `RetryInfo` entry.
+
+        Gemini's 429 body shape (per google.rpc.Status):
+            {"error": {"code": 429, "details": [
+                {"@type": ".../google.rpc.RetryInfo", "retryDelay": "53s"}, ...
+            ]}}
+        """
+        if not isinstance(body, dict):
+            return None
+        error = body.get("error")
+        if not isinstance(error, dict):
+            return None
+        details = error.get("details", [])
+        if not isinstance(details, list):
+            return None
+        for d in details:
+            if not isinstance(d, dict):
+                continue
+            if not str(d.get("@type", "")).endswith("RetryInfo"):
+                continue
+            delay = str(d.get("retryDelay", ""))
+            if delay.endswith("s"):
+                try:
+                    return float(delay[:-1])
+                except ValueError:
+                    return None
+        return None

pyetool/tool/obj_storage/__init__.py

@@ -0,0 +1,11 @@
+"""Object storage abstraction with a pluggable backend, plus a Volcano TOS implementation."""
+
+from .cli import cli
+from .factory import obj_storage
+from .volc_tos import VolcTosBackend
+
+__all__ = [
+    "VolcTosBackend",
+    "cli",  # pyproject script depends on it
+    "obj_storage",
+]

pyetool/tool/obj_storage/backend.py

@@ -0,0 +1,14 @@
+"""Protocol that any object storage backend must satisfy."""
+
+from pathlib import Path
+from typing import Protocol
+
+from pyetool.util.monad import Result
+
+
+class ObjStorageBackend(Protocol):
+    def put(self, file_path: Path | str, key: str) -> Result[str]: ...
+
+    def get(self, key: str, file_path: Path | str) -> Result[Path]: ...
+
+    def delete(self, key: str) -> Result[None]: ...

pyetool/tool/obj_storage/cli.py

@@ -0,0 +1,135 @@
+"""Click-based CLI for object storage.
+
+Pipeline-friendly: on success a single line (URL / local path / key) is printed to stdout;
+errors go to stderr with non-zero exit, so pipes break cleanly.
+"""
+
+import sys
+from pathlib import Path
+
+import click
+
+from pyetool.util.monad import Err, Ok
+
+from .factory import obj_storage
+
+
+@click.group()
+@click.option("-e", "--endpoint", default=None, help="Endpoint host. Falls back to $OBJ_STORAGE_ENDPOINT.")
+@click.option("-b", "--bucket", default=None, help="Bucket name. Falls back to $OBJ_STORAGE_BUCKET.")
+@click.option(
+    "-r", "--region", default=None, help='Region (e.g. cn-beijing). Falls back to $OBJ_STORAGE_EXTRA_ARGS["region"].'
+)
+@click.option("--backend-type", default="tos", help='Storage backend. Only "tos" is implemented for now.')
+@click.pass_context
+def cli(ctx, endpoint, bucket, region, backend_type):
+    """Upload, download, and delete files in object storage.
+
+    \b
+    Environment variables (loaded from ~/.config/pyetool/.env then ./.env):
+      TOS_ACCESS_KEY_ID         Required. Volc TOS access key ID.
+      TOS_SECRET_ACCESS_KEY     Required. Volc TOS secret access key.
+      OBJ_STORAGE_ENDPOINT      Fallback for --endpoint.
+      OBJ_STORAGE_BUCKET        Fallback for --bucket.
+      OBJ_STORAGE_EXTRA_ARGS    Semicolon-separated key:value pairs that
+                                supply backend-specific args, e.g.
+                                "region:cn-beijing;<key>:<value>". Fallback source for
+                                --region.
+
+    \b
+    Resolution order for endpoint / bucket / region: CLI flag > env var.
+    """
+    kwargs = {}
+    if region:
+        kwargs["region"] = region
+    ctx.obj = obj_storage(backend_type=backend_type, endpoint=endpoint, bucket=bucket, **kwargs)
+
+
+@cli.command()
+@click.argument("file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.argument("key")
+@click.pass_obj
+def put(backend, file: Path, key: str):
+    """Upload a local file to object storage.
+
+    \b
+    Arguments:
+      FILE  Path to the local file to upload. Must exist and be a regular file.
+      KEY   Object key (path within the bucket) the file will be stored under,
+            e.g. "videos/2026/clip.mp4". No leading slash.
+
+    \b
+    On success: the resulting public URL is printed to stdout (one line).
+    On failure: error goes to stderr, exit code is non-zero.
+
+    \b
+    Example:
+      obj-storage put ./clip.mp4 videos/2026/clip.mp4
+    """
+    match backend.put(file, key):
+        case Ok(value=url):
+            click.echo(url)
+        case Err(err=msg):
+            click.echo(f"error: {msg}", err=True)
+            sys.exit(1)
+
+
+@cli.command()
+@click.argument("key")
+@click.argument("file", type=click.Path(dir_okay=False, path_type=Path))
+@click.pass_obj
+def get(backend, key: str, file: Path):
+    """Download an object from storage to a local file.
+
+    \b
+    Arguments:
+      KEY   Object key (path within the bucket) to download,
+            e.g. "videos/2026/clip.mp4".
+      FILE  Local destination path. Existing files are overwritten;
+            the parent directory must already exist.
+
+    \b
+    On success: the local file path is printed to stdout (one line).
+    On failure: error goes to stderr, exit code is non-zero.
+
+    \b
+    Example:
+      obj-storage get videos/2026/clip.mp4 ./clip.mp4
+    """
+    match backend.get(key, file):
+        case Ok(value=path):
+            click.echo(str(path))
+        case Err(err=msg):
+            click.echo(f"error: {msg}", err=True)
+            sys.exit(1)
+
+
+@cli.command()
+@click.argument("key")
+@click.pass_obj
+def delete(backend, key: str):
+    """Delete an object from storage.
+
+    \b
+    Arguments:
+      KEY   Object key (path within the bucket) to delete,
+            e.g. "videos/2026/clip.mp4".
+
+    \b
+    On success: KEY is echoed back to stdout (useful for piping into another step).
+    On failure: error goes to stderr, exit code is non-zero.
+
+    \b
+    Example:
+      obj-storage delete videos/2026/clip.mp4
+    """
+    match backend.delete(key):
+        case Ok():
+            click.echo(key)
+        case Err(err=msg):
+            click.echo(f"error: {msg}", err=True)
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()

pyetool/tool/obj_storage/factory.py

@@ -0,0 +1,36 @@
+"""Factory that picks a concrete object-storage backend based on type and env config."""
+
+import os
+from typing import Literal
+
+from pyetool.util.env import load_env
+
+from .backend import ObjStorageBackend
+from .volc_tos import VolcTosBackend
+
+
+def obj_storage(
+    backend_type: Literal["tos"] = "tos",
+    endpoint: str | None = None,
+    bucket: str | None = None,
+    **kwargs,
+) -> ObjStorageBackend:
+    if backend_type != "tos":
+        raise ValueError("tos is the only option for now")
+
+    load_env()
+
+    if not endpoint and not (endpoint := os.environ.get("OBJ_STORAGE_ENDPOINT")):
+        raise ValueError("endpoint is not given or OBJ_STORAGE_ENDPOINT is not in .env")
+
+    if not bucket and not (bucket := os.environ.get("OBJ_STORAGE_BUCKET")):
+        raise ValueError("bucket is not given or OBJ_STORAGE_BUCKET is not in .env")
+
+    if extra_args := os.environ.get("OBJ_STORAGE_EXTRA_ARGS"):
+        # example: OBJ_STORAGE_EXTRA_ARGS=key1:value1;key2:value2...
+        kwargs.update({section.split(":")[0]: section.split(":")[1] for section in extra_args.split(";")})
+
+    if not (region := kwargs.get("region")):
+        raise ValueError("region is not provided in kwargs or in OBJ_STORAGE_EXTRA_ARGS")
+
+    return VolcTosBackend(endpoint, bucket, region=region)

pyetool/tool/obj_storage/volc_tos.py

@@ -0,0 +1,69 @@
+"""VolcEngine TOS object-storage backend."""
+
+import os
+from pathlib import Path
+
+import tos
+
+from pyetool.util.env import load_env
+from pyetool.util.monad import Err, Ok, Result
+
+
+class VolcTosBackend:
+    def __init__(self, endpoint: str, bucket: str, region: str, **_) -> None:
+        load_env()
+        self._access_key_id = os.environ.get("TOS_ACCESS_KEY_ID")
+        self._access_key_secret = os.environ.get("TOS_SECRET_ACCESS_KEY")
+
+        if not self._access_key_id:
+            raise ValueError("TOS_ACCESS_KEY_ID is not in .env")
+        if not self._access_key_secret:
+            raise ValueError("TOS_SECRET_ACCESS_KEY is not in .env")
+
+        self._endpoint = endpoint.strip().removeprefix("https://")
+        self._bucket = bucket.strip().removesuffix("/")
+        self._region = region.strip()
+
+        self._client = tos.TosClientV2(
+            ak=self._access_key_id,
+            sk=self._access_key_secret,
+            endpoint=self._endpoint,
+            region=self._region,
+        )
+
+    def put(self, file_path: Path | str, key: str) -> Result[str]:
+        file_path = Path(file_path)
+        if not file_path.exists():
+            return Err(err=f"{file_path} is not existed")
+
+        try:
+            with file_path.open("rb") as f:
+                self._client.put_object(bucket=self._bucket, key=key, content=f)
+        except Exception as e:
+            return Err(err=f"upload failed, {e}")
+
+        url = f"https://{self._bucket}.{self._endpoint}/{key}"
+        return Ok(url)
+
+    def get(self, key: str, file_path: Path | str) -> Result[Path]:
+        try:
+            content = self._client.get_object(bucket=self._bucket, key=key).read()
+        except Exception as e:
+            return Err(err=str(e))
+
+        file_path = Path(file_path)
+
+        if isinstance(content, bytes):
+            file_path.write_bytes(content)
+        else:
+            file_path.write_text(content)
+
+        return Ok(file_path)
+
+    def delete(self, key: str) -> Result[None]:
+        try:
+            self._client.delete_object(bucket=self._bucket, key=key)
+        except Exception as e:
+            return Err(err=f"delete failed: {e}")
+
+        return Ok(None)

pyetool/tool/site/__init__.py

@@ -0,0 +1,5 @@
+"""pyetool site tools: Playwright-based Page Object suites, one subpackage per site."""
+
+from .cli import cli
+
+__all__ = ["cli"]

pyetool/tool/site/bili/__init__.py

@@ -0,0 +1,7 @@
+"""Bilibili Page Object suite + CLI."""
+
+from .bili import Bili, Submission, SubmissionListPage
+from .cli import cli
+from .model import SubmissionSnapshot
+
+__all__ = ["Bili", "Submission", "SubmissionListPage", "SubmissionSnapshot", "cli"]