strands-transformers 0.2.0__tar.gz

strands_transformers-0.2.0/.github/workflows/docs.yml

@@ -0,0 +1,59 @@
+name: Docs
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "pyproject.toml"
+      - "strands_transformers/**"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+# Allow one concurrent deployment; let in-progress runs finish.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build docs
+        run: |
+          uv venv --python 3.12
+          # Docs build uses griffe's STATIC analysis for the API reference, so
+          # we only need the doc toolchain + the package importable as a path —
+          # NOT torch / transformers / strands (kept out via --no-deps).
+          uv pip install mkdocs-material "mkdocstrings[python]" pymdown-extensions
+          uv pip install --no-deps -e .
+          # Call the venv binary directly (avoid `uv run`, which would re-sync
+          # the full project and pull the heavy ML deps).
+          .venv/bin/mkdocs build --strict
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

strands_transformers-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,80 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+permissions:
+  contents: write     # create GitHub Release
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0          # full history so setuptools-scm sees the tag
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build sdist + wheel
+        run: |
+          uv venv --python 3.12
+          uv pip install build
+          # setuptools-scm derives the version from the git tag (vX.Y.Z → X.Y.Z).
+          # `python -m build` is PEP 517-isolated (only needs setuptools-scm to
+          # build), so call the venv binary directly — no heavy ML deps pulled.
+          .venv/bin/python -m build
+
+      - name: Show built artifacts
+        run: ls -l dist/
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      # Publish with the PYPI_API_TOKEN repo secret.
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
+  github-release:
+    needs: pypi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Extract version
+        id: v
+        run: echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT"
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: strands-transformers v${{ steps.v.outputs.version }}
+          generate_release_notes: true
+          files: dist/*
+          body: |
+            ## 🤗 strands-transformers v${{ steps.v.outputs.version }}
+
+            ```bash
+            uv pip install strands-transformers==${{ steps.v.outputs.version }}
+            # or
+            pip install strands-transformers==${{ steps.v.outputs.version }}
+            ```
+
+            📖 Docs: https://cagataycali.github.io/strands-transformers/

strands_transformers-0.2.0/.gitignore

@@ -0,0 +1,13 @@
+.venv
+.DS_Store
+*.bak
+__pycache__
+*_merged
+qwen3_*
+sessions
+*.egg-info
+system_prompt.prompt
+site/
+strands_transformers/_version.py
+dist/
+build/

strands_transformers-0.2.0/ARCHITECTURE.md

@@ -0,0 +1,93 @@
+# Architecture
+
+`strands-transformers` is the universal entrypoint to HuggingFace transformers for
+Strands agents — 100% task & modality coverage with zero hardcoding. It reads
+transformers' own task taxonomy at runtime, so new tasks/models work without code
+changes (the same philosophy as `use_aws` wrapping boto3 and `use_lerobot`
+wrapping lerobot).
+
+## Layout
+
+```
+strands_transformers/
+├── core/
+│   ├── registry.py   # task taxonomy + dynamic class/attr resolution
+│   ├── engine.py     # load/cache pipelines & models, device/dtype selection
+│   ├── io.py         # multimodal input coercion + JSON-safe output serialization
+│   └── compat.py     # backward-compat shims for legacy trust_remote_code models
+├── models/
+│   └── transformers.py  # TransformerModel — a Strands model provider (local brain)
+└── tools/
+    └── use_transformers.py  # the single @tool agents call
+examples/              # runnable, GPU-verified examples (see examples/README.md)
+```
+
+## Data flow
+
+```
+agent → use_transformers(action=...) ─┬─ discovery   → registry
+                                       ├─ run(task)   → engine.get_pipeline → pipeline(inputs) → io.serialize_output
+                                       └─ call(target)→ registry.resolve_attr / cached: → obj(**params) → io.serialize_output
+```
+
+### `core/registry.py` — the source of truth
+- `supported_tasks()` reads transformers' `SUPPORTED_TASKS` → `{task: {type, auto_models, default_model, pipeline_class}}`.
+- `tasks_by_modality()`, `task_info()`, `resolve_task()` (tolerant of underscores/hyphens).
+- `auto_model_classes()` lists every `Auto*` entrypoint.
+- `resolve_attr(dotted)` resolves any dotted path into transformers (class, fn,
+  method), with a root-getattr fast path so transformers' lazy `__getattr__`
+  (which raises `AttributeError` on submodule-import attempts) never aborts
+  resolution.
+- `describe(obj)` introspects signatures/docstrings for the `inspect` action.
+
+### `core/engine.py` — load, cache, run
+- `select_device()` / `select_dtype()` auto-pick cuda/mps/cpu and bf16/fp16.
+- `get_pipeline(task, model, ...)` builds & caches a `transformers.pipeline`.
+  Image-output tasks (depth-estimation, segmentation, image-to-image,
+  mask-generation) are kept in **float32** — half precision breaks PIL/numpy
+  post-processing.
+- `load_object(auto_class, model_path, ...)` loads any `AutoModel*` / `AutoProcessor`
+  / `AutoTokenizer` via `from_pretrained` for the low-level `call` layer.
+- `_CACHE` holds pipelines/models/processors keyed by `cache_key` for the session.
+
+### `core/io.py` — multimodal I/O
+- **In:** `coerce_input` decodes base64 data-URIs to PIL/bytes; paths/URLs/arrays
+  pass through natively. `decode_wav` / `maybe_decode_audio_path` pre-decode WAV
+  files for audio tasks with the stdlib `wave` module (no ffmpeg needed).
+- **Out:** `serialize_output` converts any result to JSON-safe form — audio dicts
+  → `.wav` artifacts, PIL images → `.png` artifacts, torch/numpy tensors → lists
+  (bf16/fp16 upcast to float32 first), with `_ensure_json_safe` as a final guard.
+
+### `core/compat.py` — legacy model support
+Patches transformers 4.x→5.x gaps so old `trust_remote_code` models (e.g. OpenVLA)
+run unchanged. Idempotent + re-entrant (`force=True`) because remote code can
+re-import transformers mid-load:
+- moved tokenizer symbols (`PaddingStrategy`, …) re-exposed on a real
+  file-backed `tokenization_utils` module;
+- `AutoModelForVision2Seq` recreated as an `AutoModelForImageTextToText` alias,
+  asserted everywhere `auto_map` dispatch and `register_for_auto_class()` look;
+- `tie_weights()` signature drift made kwarg-tolerant via an `init_weights` wrap;
+- broken-torchcodec detection disabled so audio pipelines take the array path;
+- `spoof_timm_version()` for models with hard timm pins.
+
+### `tools/use_transformers.py` — the one tool
+Two layers + discovery:
+- **run** — high-level pipelines (native multimodal). Folds separate images into
+  chat content for `image-text-to-text`; pre-decodes WAV for audio tasks.
+- **call** — dynamic dispatch to any class/fn/method. `cached:key[.attr]` refs
+  resolve to live cached objects (including inside `parameters`); a `"**"` param
+  key unpacks a cached mapping into kwargs (e.g. `model.predict_action(**batch)`).
+- **discovery** — `tasks`, `modalities`, `task_info`, `classes`, `inspect`,
+  `cache`, `clear_cache`, `compat`.
+
+### `models/transformers.py` — local brain
+`TransformerModel` is a Strands model provider running any local HF causal-LM as
+the agent's reasoning engine (streaming, chat templates, Qwen3 `<think>`, XML
+tool-calling). Pair it with `use_transformers` for a fully local multimodal agent.
+
+## Testing philosophy
+
+Every change is verified **end-to-end against the real implementation** — actual
+model inference / pipelines, not mocks. `examples/smoke.py` is a fast (no large
+downloads) 12-check gate across discovery + text/image/audio that exits non-zero
+on failure.

strands_transformers-0.2.0/PKG-INFO

@@ -0,0 +1,252 @@
+Metadata-Version: 2.4
+Name: strands-transformers
+Version: 0.2.0
+Summary: The universal entrypoint to HuggingFace transformers for Strands agents — 100% task & modality coverage, zero hardcoding.
+Author-email: Cagatay Cali <cagataycali@icloud.com>
+License: MIT
+Project-URL: Homepage, https://github.com/cagataycali/strands-transformers
+Project-URL: Repository, https://github.com/cagataycali/strands-transformers
+Project-URL: Issues, https://github.com/cagataycali/strands-transformers/issues
+Keywords: strands,transformers,huggingface,ai,agents,multimodal,vision,audio,video,vla,robotics,llm
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: strands-agents
+Requires-Dist: transformers>=4.40
+Requires-Dist: torch
+Requires-Dist: pillow
+Requires-Dist: numpy
+Provides-Extra: audio
+Requires-Dist: soundfile; extra == "audio"
+Requires-Dist: librosa; extra == "audio"
+Provides-Extra: vision
+Requires-Dist: pillow; extra == "vision"
+Requires-Dist: opencv-python; extra == "vision"
+Requires-Dist: av; extra == "vision"
+Provides-Extra: training
+Requires-Dist: trl; extra == "training"
+Requires-Dist: peft; extra == "training"
+Requires-Dist: accelerate; extra == "training"
+Requires-Dist: datasets; extra == "training"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings[python]; extra == "docs"
+Requires-Dist: pymdown-extensions; extra == "docs"
+Provides-Extra: all
+Requires-Dist: strands-transformers[audio,dev,docs,training,vision]; extra == "all"
+
+<div align="center">
+  <h1>🤗 Strands Transformers</h1>
+  <h3>One tool wraps <i>all</i> of HuggingFace transformers. One provider makes any local model a multimodal agent brain.</h3>
+  <p><b>Agents that see, hear, and speak — 100% task coverage, zero hardcoding, fully local.</b></p>
+
+  <div>
+    <a href="https://pypi.org/project/strands-transformers/"><img alt="pypi" src="https://img.shields.io/pypi/v/strands-transformers"/></a>
+    <a href="https://github.com/cagataycali/strands-transformers/actions/workflows/docs.yml"><img alt="docs" src="https://github.com/cagataycali/strands-transformers/actions/workflows/docs.yml/badge.svg"/></a>
+    <a href="https://github.com/cagataycali/strands-transformers/issues"><img alt="issues" src="https://img.shields.io/github/issues/cagataycali/strands-transformers"/></a>
+    <img alt="python" src="https://img.shields.io/badge/python-3.10+-blue"/>
+    <img alt="transformers" src="https://img.shields.io/badge/🤗_transformers-24_tasks-yellow"/>
+    <img alt="modalities" src="https://img.shields.io/badge/modalities-text·image·video·audio-orange"/>
+    <img alt="license" src="https://img.shields.io/badge/license-MIT-green"/>
+  </div>
+</div>
+
+---
+
+`use_aws` wraps all of boto3. `use_lerobot` wraps all of lerobot.
+**`use_transformers` wraps all of HuggingFace transformers** — every task, every
+modality, in one tool that reads transformers' own taxonomy at runtime (new task
+upstream ⇒ supported here with **no code change**). And **`TransformerModel`** makes
+any **local** HF model a drop-in Strands brain that speaks the full content-block
+protocol — image, video, audio, document. With Qwen2.5-Omni it even **speaks back**.
+
+```mermaid
+flowchart LR
+    IN["📥 text · image · video<br/>audio · document · robot-state"]
+    TOOL["🛠️ use_transformers<br/><i>tool</i>"]
+    BRAIN["🧠 TransformerModel<br/><i>local agent brain</i>"]
+    OUT["📤 text · speech · image<br/>labels · actions"]
+    IN --> TOOL --> OUT
+    IN --> BRAIN --> OUT
+    classDef i fill:#7C4DFF,stroke:#5b34d6,color:#fff;
+    classDef c fill:#FFD21E,stroke:#E68A00,color:#3a2d00;
+    classDef o fill:#00E5FF,stroke:#00b3cc,color:#003844;
+    class IN i;
+    class TOOL,BRAIN c;
+    class OUT o;
+```
+
+📖 **[Full documentation →](https://cagataycali.github.io/strands-transformers/)** &nbsp;·&nbsp; built with MkDocs (`docs/`)
+
+## Install
+
+```bash
+uv pip install strands-transformers        # from PyPI
+# or from source:
+uv pip install -e .                         # or: pip install -e .
+PYTHONPATH=. python examples/smoke.py       # verify → "12/12 checks passed"
+```
+
+<details>
+<summary>Optional extras (audio · vision · training · docs)</summary>
+
+```bash
+uv pip install -e ".[audio]"      # soundfile, librosa  (mp3/flac/ogg decode)
+uv pip install -e ".[vision]"     # opencv, av  (video)
+uv pip install -e ".[training]"   # trl, peft, accelerate
+uv pip install -e ".[docs]"       # mkdocs-material, mkdocstrings
+uv pip install -e ".[all]"        # everything
+```
+WAV audio works without extras. `device="auto"` picks cuda → mps → cpu (bf16 on GPU).
+</details>
+
+## 60-second hello — a local vision agent
+
+```python
+import io
+from PIL import Image
+from strands import Agent
+from strands_transformers import TransformerModel
+
+buf = io.BytesIO(); Image.new("RGB", (64, 64), (20, 200, 40)).save(buf, "PNG")  # green square
+
+model = TransformerModel(model_path="HuggingFaceTB/SmolVLM-256M-Instruct")
+agent = Agent(model=model, system_prompt="You are concise.")
+
+print(agent([
+    {"image": {"format": "png", "source": {"bytes": buf.getvalue()}}},
+    {"text": "Color? One word."},
+]))
+# → Green.
+```
+
+A 256M-param model in the standard Strands loop, *seeing* pixels through a content
+block — no API key, no server. Swap `model_path` for any HF VLM.
+
+## See it work
+
+Every output below is a **real** model result (CUDA · transformers 5.12 · torch 2.10):
+
+| You give it | Script | It returns |
+|-------------|--------|-----------|
+| 🖼️ a green image + "Color?" | `examples/multimodal_agent.py` | `"Green."` |
+| 🎬 brightening frames | `examples/multimodal_advanced.py` | `"BRIGHTER."` |
+| 🧰 a tool screenshot (blue) | `examples/multimodal_advanced.py` | `"Blue."` |
+| 📄 a text document | `examples/document_and_audio.py` | recovers `BANANA-42` |
+| 🔊 a 440 Hz tone (Omni) | `examples/omni_audio.py` | `"It's a pure tone."` |
+| 💬 "say: …can speak" (Omni) | `examples/omni_audio.py` | 🔊 real 24 kHz speech |
+
+▶️ **[Hear Omni speak + see all diagrams in the docs →](https://cagataycali.github.io/strands-transformers/)**
+
+## Two ways to use it
+
+<details open>
+<summary><b>As a tool</b> — <code>use_transformers</code> (discover · run · call)</summary>
+
+```python
+from strands import Agent
+from strands_transformers import use_transformers
+
+agent = Agent(tools=[use_transformers])
+agent("Transcribe recording.wav")                  # automatic-speech-recognition
+agent("What's in scene.jpg?")                       # image-text-to-text
+agent("Say 'hello from strands' as audio")          # text-to-audio
+agent("Detect objects in https://.../street.jpg")   # object-detection
+```
+
+Discover everything at runtime (`action="tasks" | "modalities" | "inspect" | …`),
+run high-level pipelines, or `call` any class/fn/method for custom models.
+→ **[The tool guide](https://cagataycali.github.io/strands-transformers/guide/the-tool/)**
+</details>
+
+<details>
+<summary><b>As the agent's brain</b> — <code>TransformerModel</code> (multimodal content blocks)</summary>
+
+Pass `image` / `video` / `audio` / `document` content blocks (and media inside a
+`toolResult`) — the provider auto-detects the model's processor and routes them.
+All outputs below are **real** results (CUDA, transformers 5.12 / torch 2.10):
+
+| Content block | Example | Verified output |
+|---|---|---|
+| `image` | `multimodal_agent.py` | `"Green."` |
+| `video` (with `fps`) | `multimodal_advanced.py` | `"BRIGHTER."` |
+| `image` in `toolResult` | `multimodal_advanced.py` | `"Blue."` |
+| `document` | `document_and_audio.py` | recovers `BANANA-42` |
+| `audio` *(our schema extension)* | `audio_content_block.py` | audio → text |
+| `audio` in **and** speech out | `omni_audio.py` | hears + **speaks** (Qwen2.5-Omni) |
+
+→ **[Agent brain](https://cagataycali.github.io/strands-transformers/guide/agent-brain/)** ·
+**[Content blocks](https://cagataycali.github.io/strands-transformers/guide/content-blocks/)** ·
+**[Audio](https://cagataycali.github.io/strands-transformers/guide/audio/)**
+</details>
+
+<details>
+<summary><b>Robotics / VLA</b> — camera + instruction → robot actions</summary>
+
+Two layers, both transformers-native and GPU-verified:
+- 🧠 **reason** — [Cosmos-Reason2-2B](https://huggingface.co/nvidia/Cosmos-Reason2-2B)
+  (a physical-AI VLM) plans over a scene via the `run` path: *"the red cube is in
+  the bottom left corner, so the arm should move there first."*
+- ⚙️ **act** — VLA models expose `predict_action` via the `call` path:
+  [MolmoAct2](https://huggingface.co/allenai/MolmoAct2-SO100_101) → `[1,30,6]`;
+  [OpenVLA-7b](https://huggingface.co/openvla/openvla-7b) → 7-DoF (auto 4.x→5.x shims).
+
+🔗 **Full agentic loop** ([`examples/robot_reason_act_agent.py`](examples/robot_reason_act_agent.py)):
+Cosmos-Reason *plans* over real RealSense frames → MolmoAct *acts* (`[1,30,6]`) —
+perception→plan→action through one tool.
+
+Lerobot-ecosystem policies (SmolVLA, π0, ACT, GR00T) use their own runtimes —
+pair with `use_lerobot`.
+→ **[Robotics guide](https://cagataycali.github.io/strands-transformers/guide/robotics/)**
+</details>
+
+## How it works
+
+Nothing is hardcoded per task — `core/registry.py` reads transformers' own
+`SUPPORTED_TASKS` at runtime, so coverage tracks upstream automatically.
+
+<details>
+<summary>Project layout</summary>
+
+```
+strands_transformers/
+├── tools/use_transformers.py   # the one @tool: discover · run · call
+├── models/transformers.py      # TransformerModel — local multimodal agent brain
+├── types/audio.py              # audio content-block extension
+└── core/{registry,engine,io,compat}.py   # taxonomy · load/cache · I/O · legacy shims
+```
+→ **[Architecture](https://cagataycali.github.io/strands-transformers/reference/architecture/)** ·
+**[API reference](https://cagataycali.github.io/strands-transformers/reference/transformer-model/)**
+</details>
+
+## Examples
+
+12 runnable, GPU-verified examples in [`examples/`](examples/) — image, video,
+audio, document, Omni speech, VLA, and pipelines. Run any:
+
+```bash
+PYTHONPATH=. python examples/<name>.py
+```
+
+→ **[Examples & FAQ](https://cagataycali.github.io/strands-transformers/reference/examples/)**
+
+## License
+
+MIT — built with [Strands Agents SDK](https://github.com/strands-agents/sdk-python)
+and [HuggingFace Transformers](https://github.com/huggingface/transformers).
+
+<div align="center">
+  <sub>If this saved you a pile of per-model glue code, consider giving it a ⭐</sub>
+</div>