teich 0.1.1a1__tar.gz

teich-0.1.1a1/.github/workflows/# Agent Traces on the Hub.md

@@ -0,0 +1,67 @@
+# Agent Traces on the Hub
+
+[Hugging Face](https://huggingface.co/)
+- [Models](https://huggingface.co/models)
+- [Datasets](https://huggingface.co/datasets)
+- [Spaces](https://huggingface.co/spaces)
+- [Buckets new](https://huggingface.co/storage)
+- [Docs](https://huggingface.co/docs)
+- [Enterprise](https://huggingface.co/enterprise)
+- [Pricing](https://huggingface.co/pricing)
+-     
+- 
+- [Log In](https://huggingface.co/login)
+- [Sign Up](https://huggingface.co/join)
+[Models](https://huggingface.co/models)
+[Datasets](https://huggingface.co/datasets)
+[Spaces](https://huggingface.co/spaces)
+[Buckets new](https://huggingface.co/storage)
+[Docs](https://huggingface.co/docs)
+[Enterprise](https://huggingface.co/enterprise)
+[Pricing](https://huggingface.co/pricing)
+[Log In](https://huggingface.co/login)
+[Sign Up](https://huggingface.co/join)
+[Back to Changelog](https://huggingface.co/changelog)
+[Upvote 98](https://huggingface.co/login?next=%2Fchangelog%2Fagent-trace-viewer)
+- https://huggingface.co/julien-c
+- https://huggingface.co/clem
+- https://huggingface.co/lhoestq
+- https://huggingface.co/thomasgauthier
+- https://huggingface.co/lewtun
+- +93
+
+# Agent Traces on the Hub
+[Upvote 98](https://huggingface.co/login?next=%2Fchangelog%2Fagent-trace-viewer)
+- https://huggingface.co/julien-c
+- https://huggingface.co/clem
+- https://huggingface.co/lhoestq
+- https://huggingface.co/thomasgauthier
+- https://huggingface.co/lewtun
+- +93
+You can now upload traces from your agents (Claude Code, Codex, Pi) directly to Hugging Face Datasets. The Hub auto-detects trace formats and tags your dataset as [Traces](https://huggingface.co/datasets?format=format:agent-traces&sort=trending), with a dedicated viewer for browsing sessions, turns, tool calls, and model responses.
+No preprocessing needed, just upload the JSONL files from your local session directories as-is:
+
+```
+~/.claude/projects
+```
+
+
+```
+~/.codex/sessions
+```
+
+
+```
+~/.pi/agent/sessions
+```
+
+Useful for sharing debugging workflows, benchmarking agent behavior across models, or building training data from real coding sessions.
+[TOS](https://huggingface.co/terms-of-service)
+[Privacy](https://huggingface.co/privacy)
+[About](https://huggingface.co/huggingface)
+[Careers](https://apply.workable.com/huggingface/)
+[Models](https://huggingface.co/models)
+[Datasets](https://huggingface.co/datasets)
+[Spaces](https://huggingface.co/spaces)
+[Pricing](https://huggingface.co/pricing)
+[Docs](https://huggingface.co/docs)

teich-0.1.1a1/.github/workflows/test.yml

@@ -0,0 +1,81 @@
+name: Tests
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          cd v2
+          uv pip install -e ".[dev]"
+
+      - name: Run unit tests
+        run: |
+          cd v2
+          pytest tests/ -v -m "not integration" --tb=short
+
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        run: |
+          cd v2
+          docker build -f docker/codex-runtime.Dockerfile -t agentic-datagen-codex:v2 .
+
+      - name: Test Codex CLI in container
+        run: |
+          docker run --rm agentic-datagen-codex:v2 codex --version
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: [unit-tests, docker-build]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    # Only run on main to avoid burning API credits on every PR
+    environment: integration-tests
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          cd v2
+          uv pip install -e ".[dev]"
+
+      - name: Run integration tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          cd v2
+          pytest tests/test_integration.py -v --tb=short

teich-0.1.1a1/.gitignore

@@ -0,0 +1,22 @@
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+.venv/
+venv/
+.pytest_cache/
+.benchmarks/
+.deepeval/
+.mypy_cache/
+.ruff_cache/
+output/
+sandbox/
+test_run/
+outputs/
+qwen_lora/
+traces-test/
+*.env
+*.env.*
+.DS_Store
+Thumbs.db

teich-0.1.1a1/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: teich
+Version: 0.1.1a1
+Summary: Generate agent training data from Codex and Pi traces
+License: Apache-2.0
+Requires-Python: >=3.10
+Requires-Dist: datasets>=2.19.0
+Requires-Dist: docker>=7.0
+Requires-Dist: huggingface-hub>=0.23.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: respx>=0.22; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Teich
+
+`v2/` is the experimental trace-first package for collecting raw agent sessions and converting them into training-ready data.
+
+## What it does today
+
+- Runs Codex and Pi in a shared Docker runtime with `uv`, `npm`, `@openai/codex`, and `@mariozechner/pi-coding-agent`
+- Configures Codex through a mounted `CODEX_HOME/config.toml`
+- Configures Pi through an isolated mounted `~/.pi/agent/settings.json`
+- Exports raw session traces from mounted Codex and Pi session directories
+- Writes a trace-folder `README.md` for upload
+- Exposes Python conversion helpers for training data preparation
+
+## Usage
+
+```bash
+# Initialize a project
+uvx teich init my-project
+cd my-project
+
+# Run with the configured agent provider and model settings
+uvx teich generate -c config.yaml
+```
+
+## Local OSS providers
+
+If you want Codex to talk to a local provider like LM Studio or Ollama, set the provider in config or env:
+
+```powershell
+$env:TEICH_PROVIDER='LMstudio'
+$env:TEICH_MODEL='gemma-4'
+$env:TEICH_API_KEY='llm'
+$env:TEICH_BASE_URL='http://localhost:1234/v1'
+python -m teich generate -c test_run/config.yaml
+```
+
+`v2` maps `LMstudio` and `ollama` onto Codex's native `--oss --local-provider ...` flow.
+
+## Configuration model
+
+Important fields in `config.yaml`:
+
+```yaml
+agent:
+  provider: codex  # or pi
+
+model:
+  model: codex-mini-latest
+  approval_policy: never
+  sandbox: danger-full-access
+  reasoning_effort: null
+
+api:
+  provider: openai
+  base_url: null
+  api_key: null
+```
+
+Legacy `model.approval_mode` is still accepted and normalized internally.
+
+## Python conversion API
+
+```python
+from pathlib import Path
+from teich import convert_traces_to_training_data
+
+examples = convert_traces_to_training_data(Path("./output"))
+```
+
+The converter currently maps example-style raw traces into message/tool records with:
+
+- system/developer instructions
+- user messages
+- assistant messages
+- `reasoning_content`
+- tool calls
+- tool results
+
+## Development
+
+```bash
+uv pip install -e ".[dev]"
+pytest tests/test_config.py tests/test_cli.py tests/test_runner.py -q
+```
+
+## Architecture
+
+- **Shared Docker runtime**: container image includes Node.js, `uv`, `uvx`, `@openai/codex`, and `@mariozechner/pi-coding-agent`
+- **Isolated Pi config**: Pi runs with a mounted per-run `~/.pi/agent` directory inside the container
+- **Codex config**: generated `config.toml` under a mounted `CODEX_HOME`
+- **Session export**: raw JSONL sessions are copied from mounted Codex or Pi session storage into the user output directory
+- **Upload-first output**: traces are preserved in raw form before later conversion
+- **Provider-aware boundary**: `agent.provider` selects either the Codex or Pi raw-trace path
+
+## Project Structure
+
+```text
+v2/
+├── docker/
+│   └── codex-runtime.Dockerfile
+├── src/teich/
+│   ├── __init__.py
+│   ├── __main__.py
+│   ├── cli.py
+│   ├── config.py
+│   ├── converter.py
+│   ├── runner.py
+│   └── trace_readme.py
+└── tests/
+    ├── test_cli.py
+    ├── test_config.py
+    └── test_runner.py

teich-0.1.1a1/README.md

@@ -0,0 +1,112 @@
+# Teich
+
+`v2/` is the experimental trace-first package for collecting raw agent sessions and converting them into training-ready data.
+
+## What it does today
+
+- Runs Codex and Pi in a shared Docker runtime with `uv`, `npm`, `@openai/codex`, and `@mariozechner/pi-coding-agent`
+- Configures Codex through a mounted `CODEX_HOME/config.toml`
+- Configures Pi through an isolated mounted `~/.pi/agent/settings.json`
+- Exports raw session traces from mounted Codex and Pi session directories
+- Writes a trace-folder `README.md` for upload
+- Exposes Python conversion helpers for training data preparation
+
+## Usage
+
+```bash
+# Initialize a project
+uvx teich init my-project
+cd my-project
+
+# Run with the configured agent provider and model settings
+uvx teich generate -c config.yaml
+```
+
+## Local OSS providers
+
+If you want Codex to talk to a local provider like LM Studio or Ollama, set the provider in config or env:
+
+```powershell
+$env:TEICH_PROVIDER='LMstudio'
+$env:TEICH_MODEL='gemma-4'
+$env:TEICH_API_KEY='llm'
+$env:TEICH_BASE_URL='http://localhost:1234/v1'
+python -m teich generate -c test_run/config.yaml
+```
+
+`v2` maps `LMstudio` and `ollama` onto Codex's native `--oss --local-provider ...` flow.
+
+## Configuration model
+
+Important fields in `config.yaml`:
+
+```yaml
+agent:
+  provider: codex  # or pi
+
+model:
+  model: codex-mini-latest
+  approval_policy: never
+  sandbox: danger-full-access
+  reasoning_effort: null
+
+api:
+  provider: openai
+  base_url: null
+  api_key: null
+```
+
+Legacy `model.approval_mode` is still accepted and normalized internally.
+
+## Python conversion API
+
+```python
+from pathlib import Path
+from teich import convert_traces_to_training_data
+
+examples = convert_traces_to_training_data(Path("./output"))
+```
+
+The converter currently maps example-style raw traces into message/tool records with:
+
+- system/developer instructions
+- user messages
+- assistant messages
+- `reasoning_content`
+- tool calls
+- tool results
+
+## Development
+
+```bash
+uv pip install -e ".[dev]"
+pytest tests/test_config.py tests/test_cli.py tests/test_runner.py -q
+```
+
+## Architecture
+
+- **Shared Docker runtime**: container image includes Node.js, `uv`, `uvx`, `@openai/codex`, and `@mariozechner/pi-coding-agent`
+- **Isolated Pi config**: Pi runs with a mounted per-run `~/.pi/agent` directory inside the container
+- **Codex config**: generated `config.toml` under a mounted `CODEX_HOME`
+- **Session export**: raw JSONL sessions are copied from mounted Codex or Pi session storage into the user output directory
+- **Upload-first output**: traces are preserved in raw form before later conversion
+- **Provider-aware boundary**: `agent.provider` selects either the Codex or Pi raw-trace path
+
+## Project Structure
+
+```text
+v2/
+├── docker/
+│   └── codex-runtime.Dockerfile
+├── src/teich/
+│   ├── __init__.py
+│   ├── __main__.py
+│   ├── cli.py
+│   ├── config.py
+│   ├── converter.py
+│   ├── runner.py
+│   └── trace_readme.py
+└── tests/
+    ├── test_cli.py
+    ├── test_config.py
+    └── test_runner.py

teich-0.1.1a1/ROADMAP.md

@@ -0,0 +1,165 @@
+# Teich - Roadmap
+
+## Current Status: Functional Experimental Baseline
+- Docker runtime with Codex, uv, npm
+- Configuration system with YAML models and MCP server definitions
+- `python -m teich` and CLI init/generate commands
+- Raw session extraction from mounted `CODEX_HOME/sessions`
+- Auto-generated trace-folder README
+- Importable converter for raw Codex traces to training-style messages/tools
+- Focused unit coverage for config, CLI, and runner behavior
+
+---
+
+## Phase 1: Testing & Hardening (Next)
+
+### 1.1 Integration Testing
+- [x] Test actual Docker image build
+- [ ] Validate live `codex exec` end-to-end against installed Codex CLI versions
+- [ ] Verify session files are extracted correctly with real sessions
+- [ ] Verify trace format matches HF expectations on actual generated traces
+- [ ] Test MCP server configuration with real servers
+- [ ] Validate LM Studio / Ollama local-provider runs through Codex OSS mode
+
+### 1.2 Error Handling
+- [ ] Handle Docker not installed/running
+- [ ] Handle invalid OpenAI API key or unavailable local provider
+- [x] Handle network timeouts during Codex execution
+- [x] Handle session extraction failures
+- [ ] Retry logic for failed prompts
+
+### 1.3 Output Format Validation
+- [x] Validate trace JSONL structure against example traces
+- [ ] Ensure HF trace viewer compatibility
+- [x] Generate README for trace upload directories
+
+---
+
+## Phase 2: Training Data Conversion
+
+### 2.1 Converter Module
+Implemented `src/teich/converter.py`:
+
+```python
+from teich import convert_traces_to_training_data, TrainingExample
+
+examples = convert_traces_to_training_data(
+    traces_dir=Path("./output")
+)
+```
+
+### 2.2 Supported Formats
+- [x] **OpenAI-style chat/message format** (primary internal output):
+  ```json
+  {
+    "messages": [
+      {"role": "system", "content": "..."},
+      {"role": "user", "content": "..."},
+      {"role": "assistant", "content": "...", "reasoning_content": "..."},
+      {"role": "assistant", "tool_calls": [...]},
+      {"role": "tool", "tool_call_id": "...", "content": "..."}
+    ]
+  }
+```
+- [ ] **Anthropic Messages API**
+- [ ] **Gemini Format**
+
+### 2.3 Field Mapping
+From Codex traces to training examples:
+- [x] Extract system prompts from session init / developer messages
+- [x] Map `message` events → user/assistant messages
+- [x] Extract `reasoning_content` from reasoning summary events
+- [x] Map `function_call` → assistant message with tool_calls
+- [x] Map `function_call_output` → tool message
+- [ ] Extract tool schemas when present in raw traces
+- [x] Handle multi-turn conversations correctly
+
+---
+
+## Phase 3: Advanced Features
+
+### 3.1 Parallel Execution
+- [ ] Run multiple prompts concurrently
+- [ ] Configurable max_workers
+- [ ] Progress tracking for batch jobs
+
+### 3.2 Session Resumption
+- [ ] Save progress checkpoint
+- [ ] Resume interrupted runs
+- [ ] Skip already-completed prompts
+
+### 3.3 Output Formats
+- [ ] Hugging Face datasets integration
+- [ ] Parquet output option
+- [ ] Train/validation split generation
+
+### 3.4 Quality Filtering
+- [ ] Filter empty/short sessions
+- [ ] Detect failed/error sessions
+- [ ] Workspace artifact validation
+- [ ] Configurable quality thresholds
+
+---
+
+## Phase 4: Extended Model Support
+
+### 4.1 OpenRouter/OpenAI-Compatible APIs
+Since Codex CLI behavior varies by provider, keep exploring alternatives:
+- [ ] Research: Can we use `aider` or similar tools?
+- [ ] Custom runner for generic OpenAI-compatible APIs
+- [x] OpenRouter/config override path in current Codex runner
+- [ ] Harden compatibility for non-OpenAI endpoints under real runs
+
+### 4.2 Multi-Provider Support
+- [x] Modular config boundary with `agent.provider`
+- [ ] Pi agent runner
+- [ ] Anthropic Claude runner
+- [ ] Google Gemini runner
+- [ ] Ollama/local model runner beyond Codex OSS mode
+
+---
+
+## Phase 5: Production Polish
+
+### 5.1 Documentation
+- [ ] Full API documentation
+- [ ] Tutorial: Creating your first dataset
+- [ ] Tutorial: Fine-tuning with generated data
+- [ ] Example configs for common use cases
+
+### 5.2 CLI Improvements
+- [ ] `validate` command to check config
+- [ ] `preview` command to see what would be generated
+- [ ] `status` command to check previous runs
+- [ ] Rich progress bars and logging
+
+### 5.3 Testing
+- [ ] Integration tests with real API calls (mocked)
+- [ ] Docker build tests in CI
+- [ ] Format validation tests
+- [ ] End-to-end workflow tests
+
+---
+
+## Immediate Next Steps
+
+1. **Validate live Codex execution with the installed CLI**:
+   ```bash
+   cd v2
+   python -m teich generate -c test_run/config.yaml
+   ```
+
+2. **Verify trace files** are generated and match the example-style raw session format
+
+3. **Inspect an actual generated trace** and tighten converter field mapping if needed
+
+4. **Prototype Pi-agent trace ingestion** behind the same conversion/export interfaces
+
+---
+
+## Open Questions
+
+1. Which Codex CLI versions should `v2` explicitly support for non-interactive runs?
+2. Should LM Studio and Ollama stay routed through Codex OSS mode, or get their own non-Codex runner?
+3. What quality metrics should we filter on?
+4. How should we handle tool schemas that vary between providers?

teich-0.1.1a1/docker/codex-runtime.Dockerfile

@@ -0,0 +1,52 @@
+# syntax=docker/dockerfile:1
+FROM node:22-slim
+
+# Install system dependencies with cache mount and minimal packages
+# Removed: build-essential (only needed for compiling, not runtime)
+# Added: --no-install-recommends to skip extra packages
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+    curl \
+    ca-certificates \
+    python3 \
+    python3-dev \
+    python3-minimal \
+    python3-pip \
+    python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN ln -sf /usr/bin/python3 /usr/local/bin/python && \
+    python3 -m venv /opt/venv && \
+    /opt/venv/bin/python -m pip install --upgrade pip setuptools wheel && \
+    ln -sf /opt/venv/bin/pip /usr/local/bin/pip && \
+    ln -sf /opt/venv/bin/pip3 /usr/local/bin/pip3
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+ENV VIRTUAL_ENV=/opt/venv
+
+# Install Astral uv and @openai/codex in one layer
+# Use npm cache mount for faster installs
+RUN --mount=type=cache,target=/root/.npm \
+    mkdir -p ${PLAYWRIGHT_BROWSERS_PATH} && \
+    curl -LsSf https://astral.sh/uv/install.sh | sh && \
+    mv /root/.local/bin/uv /usr/local/bin/uv && \
+    mv /root/.local/bin/uvx /usr/local/bin/uvx && \
+    npm install -g @openai/codex @mariozechner/pi-coding-agent playwright && \
+    npx playwright install --with-deps chromium && \
+    node --version && npm --version && npx --version && uv --version && uvx --version && python --version && python3 --version && pip --version && pip3 --version && codex --version && pi --version
+
+# Create working directory and user in one layer
+WORKDIR /workspace
+RUN useradd -m -s /bin/bash codex && \
+    mkdir -p /home/codex/.codex/sessions && \
+    chown -R codex:codex /home/codex /workspace ${PLAYWRIGHT_BROWSERS_PATH} ${VIRTUAL_ENV}
+
+USER codex
+ENV CODEX_HOME=/home/codex
+ENV HOME=/home/codex
+ENV NODE_PATH="/usr/local/lib/node_modules"
+ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH"
+
+CMD ["bash"]