rl-expert-skills 1.0.0

package/README.md

@@ -0,0 +1,65 @@
+# rl-expert-skills
+
+45 reinforcement learning skills for **Claude Code** and **OpenAI Codex**.
+
+Covers architecture, algorithms, environments, evaluation, and MLOps.
+
+## Usage
+
+From any project folder:
+
+```bash
+npx rl-expert-skills
+```
+
+The installer will ask:
+
+1. **Which tool?** — Claude Code, OpenAI Codex, or both
+2. **Scope?** — current project or current user
+
+### What gets installed
+
+| Tool | Project scope | User scope |
+|------|--------------|------------|
+| Claude Code | `.claude/skills/` | `~/.claude/skills/` |
+| OpenAI Codex | `.codex/skills/` + `AGENTS.md` | `~/.codex/skills/` + `~/.codex/instructions.md` |
+
+## Skills included
+
+**Orchestrator agents**
+- `rl-architect` — MDP formulation, algorithm selection, system blueprint
+- `rl-algo-engineer` — neural networks, training loop, loss functions
+- `rl-env-engineer` — Gymnasium/PettingZoo environments and wrappers
+- `rl-evaluator` — evaluation, OOD testing, failure diagnosis
+- `rl-mlops-engineer` — experiment tracking, distributed training, checkpointing
+
+**Individual skills** (40 skills covering reward design, policy/critic architecture, MARL, offline RL, RLHF, domain randomization, and more)
+
+## Publishing
+
+### First time
+
+```bash
+cd F:/rl-expert-skills
+npm login
+npm publish --access public
+```
+
+### Updating skills
+
+After editing skills in `F:\RL-Expert\skills\`, re-bundle and republish:
+
+```bash
+cd F:/rl-expert-skills
+npm run prepare       # re-copies skills from ../RL-Expert/skills/
+npm version patch     # bump version (patch / minor / major)
+npm publish --access public
+```
+
+### Using before publishing (local test)
+
+From any project folder:
+
+```bash
+npx F:/rl-expert-skills
+```

package/bin/cli.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+
+import { checkbox, select, confirm } from '@inquirer/prompts';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { installClaude } from '../lib/claude.js';
+import { installCodex } from '../lib/codex.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const SKILLS_DIR = path.join(__dirname, '..', 'skills');
+
+async function main() {
+  console.log('\nRL Expert Skills Installer');
+  console.log('==========================');
+  console.log('45 reinforcement learning skills for Claude Code and OpenAI Codex\n');
+
+  const tools = await checkbox({
+    message: 'Which tool(s) do you want to install the skills for?',
+    choices: [
+      { name: 'Claude Code', value: 'claude', checked: true },
+      { name: 'OpenAI Codex', value: 'codex', checked: false },
+    ],
+    validate: (choices) => choices.length > 0 || 'Please select at least one tool.',
+  });
+
+  const scope = await select({
+    message: 'Install scope?',
+    choices: [
+      {
+        name: 'Current project  (.claude/skills/ or .codex/skills/)',
+        value: 'project',
+      },
+      {
+        name: 'Current user     (~/.claude/skills/ or ~/.codex/skills/)',
+        value: 'user',
+      },
+    ],
+  });
+
+  console.log('');
+
+  const results = [];
+
+  for (const tool of tools) {
+    if (tool === 'claude') {
+      const result = await installClaude(SKILLS_DIR, scope);
+      results.push(result);
+    } else if (tool === 'codex') {
+      const result = await installCodex(SKILLS_DIR, scope);
+      results.push(result);
+    }
+  }
+
+  console.log('\nDone');
+  console.log('----');
+  for (const r of results) {
+    console.log(`  ${r.tool}: ${r.count} skills -> ${r.path}`);
+    if (r.extra) console.log(`  ${r.extra}`);
+  }
+  console.log('');
+}
+
+main().catch((err) => {
+  console.error('\nError:', err.message);
+  process.exit(1);
+});

package/lib/claude.js

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { copyDir } from './copy.js';
+
+/**
+ * Install skills for Claude Code.
+ *
+ * Project scope: <cwd>/.claude/skills/<skill>/
+ * User scope:    ~/.claude/skills/<skill>/
+ */
+export async function installClaude(skillsDir, scope) {
+  const targetDir =
+    scope === 'project'
+      ? path.join(process.cwd(), '.claude', 'skills')
+      : path.join(os.homedir(), '.claude', 'skills');
+
+  fs.mkdirSync(targetDir, { recursive: true });
+
+  const skills = fs.readdirSync(skillsDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+
+  for (const skill of skills) {
+    const src = path.join(skillsDir, skill);
+    const dest = path.join(targetDir, skill);
+    copyDir(src, dest);
+  }
+
+  process.stdout.write(`  Claude Code  installed ${skills.length} skills\n`);
+
+  return {
+    tool: 'Claude Code ',
+    count: skills.length,
+    path: targetDir,
+  };
+}

package/lib/codex.js

@@ -0,0 +1,108 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { copyDir, readSkillMeta } from './copy.js';
+
+const AGENTS_HEADER = '<!-- rl-expert-skills -->';
+const AGENTS_FOOTER = '<!-- /rl-expert-skills -->';
+
+/**
+ * Install skills for OpenAI Codex.
+ *
+ * Project scope:
+ *   - Skills copied to <cwd>/.codex/skills/<skill>/
+ *   - <cwd>/AGENTS.md updated with skill index block
+ *
+ * User scope:
+ *   - Skills copied to ~/.codex/skills/<skill>/
+ *   - ~/.codex/instructions.md updated with skill index block
+ */
+export async function installCodex(skillsDir, scope) {
+  const isProject = scope === 'project';
+  const baseDir = isProject ? process.cwd() : os.homedir();
+  const targetDir = path.join(baseDir, '.codex', 'skills');
+
+  fs.mkdirSync(targetDir, { recursive: true });
+
+  const skills = fs.readdirSync(skillsDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+
+  for (const skill of skills) {
+    const src = path.join(skillsDir, skill);
+    const dest = path.join(targetDir, skill);
+    copyDir(src, dest);
+  }
+
+  // Build the skill index block to inject into AGENTS.md / instructions.md
+  const skillLines = skills.map((skill) => {
+    const meta = readSkillMeta(path.join(skillsDir, skill));
+    const label = meta?.name || skill;
+    const desc = meta?.description
+      ? ` — ${meta.description.slice(0, 120)}${meta.description.length > 120 ? '…' : ''}`
+      : '';
+    return `- **${label}** (\`.codex/skills/${skill}/SKILL.md\`)${desc}`;
+  });
+
+  const relPath = isProject ? '.codex/skills' : '~/.codex/skills';
+  const block = [
+    AGENTS_HEADER,
+    '## RL Expert Skills',
+    '',
+    `The following reinforcement learning skills are installed in \`${relPath}\`.`,
+    'Reference a skill by name to load its instructions into your context.',
+    '',
+    ...skillLines,
+    AGENTS_FOOTER,
+  ].join('\n');
+
+  // Determine target instructions file
+  const instructionsFile = isProject
+    ? path.join(baseDir, 'AGENTS.md')
+    : path.join(baseDir, '.codex', 'instructions.md');
+
+  if (!isProject) {
+    fs.mkdirSync(path.join(baseDir, '.codex'), { recursive: true });
+  }
+
+  upsertBlock(instructionsFile, block);
+
+  const extra = `updated ${isProject ? 'AGENTS.md' : '~/.codex/instructions.md'}`;
+  process.stdout.write(`  OpenAI Codex installed ${skills.length} skills\n`);
+
+  return {
+    tool: 'OpenAI Codex',
+    count: skills.length,
+    path: targetDir,
+    extra,
+  };
+}
+
+/**
+ * Insert or replace the rl-expert-skills block in a markdown file.
+ */
+function upsertBlock(filePath, block) {
+  let existing = '';
+  if (fs.existsSync(filePath)) {
+    existing = fs.readFileSync(filePath, 'utf8');
+  }
+
+  const startIdx = existing.indexOf(AGENTS_HEADER);
+  const endIdx = existing.indexOf(AGENTS_FOOTER);
+
+  let updated;
+  if (startIdx !== -1 && endIdx !== -1) {
+    // Replace existing block
+    updated =
+      existing.slice(0, startIdx) +
+      block +
+      existing.slice(endIdx + AGENTS_FOOTER.length);
+  } else {
+    // Append block
+    updated = existing
+      ? existing.trimEnd() + '\n\n' + block + '\n'
+      : block + '\n';
+  }
+
+  fs.writeFileSync(filePath, updated, 'utf8');
+}

package/lib/copy.js

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Recursively copy a directory from src to dest.
+ * Overwrites existing files.
+ */
+export function copyDir(src, dest) {
+  fs.mkdirSync(dest, { recursive: true });
+  for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, entry.name);
+    const destPath = path.join(dest, entry.name);
+    if (entry.isDirectory()) {
+      copyDir(srcPath, destPath);
+    } else {
+      fs.copyFileSync(srcPath, destPath);
+    }
+  }
+}
+
+/**
+ * Read SKILL.md in a skill directory and return its frontmatter fields.
+ * Returns { name, description } or null on failure.
+ */
+export function readSkillMeta(skillDir) {
+  const skillMdPath = path.join(skillDir, 'SKILL.md');
+  if (!fs.existsSync(skillMdPath)) return null;
+  const content = fs.readFileSync(skillMdPath, 'utf8');
+  const match = content.match(/^---\n([\s\S]*?)\n---/);
+  if (!match) return null;
+  const front = match[1];
+  const name = (front.match(/^name:\s*(.+)$/m) || [])[1]?.trim();
+  // description may be single-line or quoted multi-line — grab everything after "description: "
+  const descMatch = front.match(/^description:\s*(['"]?)([\s\S]*?)(\1)\s*(?:\n\w|$)/m);
+  const description = descMatch ? descMatch[2].replace(/\n\s*/g, ' ').trim() : '';
+  return { name, description };
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "rl-expert-skills",
+  "version": "1.0.0",
+  "description": "RL Expert skills for Claude Code and OpenAI Codex — 45 reinforcement learning skills covering architecture, algorithms, environments, evaluation, and MLOps.",
+  "type": "module",
+  "bin": {
+    "rl-expert-skills": "./bin/cli.js"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "skills/"
+  ],
+  "scripts": {
+    "prepare": "node scripts/copy-skills.js"
+  },
+  "dependencies": {
+    "@inquirer/prompts": "^7.5.0"
+  },
+  "devDependencies": {},
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "reinforcement-learning",
+    "rl",
+    "claude",
+    "codex",
+    "skills",
+    "ai"
+  ],
+  "license": "MIT"
+}

package/skills/analyzing-saliency-and-values/SKILL.md

@@ -0,0 +1,83 @@
+---
+name: Analyze Saliency and Values
+description: This skill should be used when the user asks to "analyze saliency", "compute feature saliency", "run Grad-CAM", "apply integrated gradients", "extract attention weights", "visualize what the agent looks at", "inspect temporal attention maps", or "check if the memory architecture is useful". Do NOT hallucinate parameters outside the boundaries of Analyze Saliency and Values.
+version: 0.1.0
+---
+
+# Analyze Saliency and Values
+
+Isolate attention and feature saliency using Grad-CAM or Integrated Gradients to reveal which inputs drive the agent's decisions and whether the memory architecture is earning its compute.
+
+## Step 1 — Gradient Saliency Extraction
+
+Push a specific observation forward through the Actor network and backpropagate gradients to the input with respect to the chosen continuous action magnitude:
+
+```python
+obs_tensor = torch.tensor(obs, requires_grad=True)
+action = actor(obs_tensor)
+action[target_dim].backward()
+saliency = obs_tensor.grad.abs()
+```
+
+For image-based policies, apply Grad-CAM to the final convolutional feature map:
+
+```python
+# See scripts/grad_cam.py for full implementation
+gradients = []  # hooked during backward
+activations = []  # hooked during forward
+cam = (gradients * activations).mean(dim=1).relu()
+cam = F.interpolate(cam, size=obs.shape[-2:], mode='bilinear')
+```
+
+Overlay the resulting heatmap on the original frame to highlight which pixels or vector indices drove the decision most.
+
+For continuous vector observations, produce a bar chart of per-dimension gradient magnitudes ranked by importance.
+
+## Step 2 — Temporal Attention Weights
+
+If the Algo Engineer implemented a Transformer or LSTM, extract the attention map matrix:
+
+**Transformer attention:**
+```python
+# Hook into multi-head attention output_weights
+attn_weights = model.transformer.layers[i].self_attn(..., need_weights=True)
+# Shape: (batch, heads, seq_len, seq_len)
+```
+
+**LSTM hidden state analysis:**
+```python
+# Track hidden state norm over time — large norm = high reliance on that step
+hidden_norms = [h.norm().item() for h in hidden_sequence]
+```
+
+**Decision rule — evaluate temporal range:**
+- If the agent relies exclusively on the instant frame (`attn_weights[:, :, -1, -1] ≈ 1.0`), the memory architecture is useless.
+- Inform the Architect: the LSTM/Transformer should be replaced with a simple MLP to save compute and reduce variance.
+- Document the finding in the saliency report with the attention matrix visualized as a heatmap.
+
+## Step 3 — Integrated Gradients (Baseline Comparison)
+
+Integrated Gradients attribute importance relative to a neutral baseline (e.g., zero observation):
+
+$$\text{IG}_i(x) = (x_i - x'_i) \times \int_{\alpha=0}^{1} \frac{\partial F(x' + \alpha(x - x'))}{\partial x_i} d\alpha$$
+
+Approximate with 50 interpolation steps. See `scripts/integrated_gradients.py` for the full loop.
+
+Useful for checking whether the agent responds to physically meaningful features or spurious correlations.
+
+## Step 4 — Report
+
+Produce a **Saliency Analysis Report** containing:
+1. Grad-CAM / saliency heatmaps for at least 5 representative states (goal-approach, obstacle-avoidance, failure state)
+2. Temporal attention matrix visualization (if memory architecture present)
+3. Verdict on memory architecture utility
+4. Top-5 most influential observation dimensions with physical interpretation
+
+## Additional Resources
+
+### Reference Files
+- **`references/saliency_math.md`** — Grad-CAM and Integrated Gradients derivations, formulas, and edge cases
+
+### Scripts
+- **`scripts/grad_cam.py`** — Full Grad-CAM implementation with forward/backward hooks
+- **`scripts/integrated_gradients.py`** — Integrated Gradients loop with interpolation

package/skills/analyzing-saliency-and-values/references/saliency_math.md

@@ -0,0 +1,48 @@
+# Saliency Math Reference
+
+## Grad-CAM Derivation
+
+Grad-CAM (Gradient-weighted Class Activation Mapping) computes importance weights for each feature map channel $k$ by global-average-pooling the gradients of the target output $y^c$ with respect to the feature map activations $A^k$:
+
+$$\alpha_k^c = \frac{1}{Z} \sum_i \sum_j \frac{\partial y^c}{\partial A_{ij}^k}$$
+
+The class activation map is then:
+
+$$L_{\text{Grad-CAM}}^c = \text{ReLU}\left(\sum_k \alpha_k^c A^k\right)$$
+
+The ReLU is critical: only features with a positive influence on the target class are retained.
+
+**For RL actor networks:** Replace the class score $y^c$ with the action magnitude $\pi_\theta(s)[\text{dim}]$ or the Q-value $Q(s, a)$.
+
+## Integrated Gradients Derivation
+
+Integrated Gradients (IG) compute the contribution of each input feature $x_i$ relative to a baseline $x'$ (typically zeros or Gaussian noise):
+
+$$\text{IG}_i(x) = (x_i - x'_i) \times \int_{\alpha=0}^{1} \frac{\partial F(x' + \alpha(x - x'))}{\partial x_i} \, d\alpha$$
+
+**Numerical approximation (Riemann sum, 50 steps):**
+
+$$\text{IG}_i(x) \approx (x_i - x'_i) \times \sum_{k=1}^{m} \frac{\partial F\left(x' + \frac{k}{m}(x - x')\right)}{\partial x_i} \times \frac{1}{m}$$
+
+IG satisfies the **completeness axiom**: $\sum_i \text{IG}_i(x) = F(x) - F(x')$. This means attributions sum to the actual output difference — a correctness guarantee Grad-CAM lacks.
+
+## Transformer Attention Extraction
+
+Multi-head attention computes:
+
+$$\text{Attention}(Q, K, V) = \text{softmax}\left(\frac{QK^T}{\sqrt{d_k}}\right)V$$
+
+The attention weight matrix $W = \text{softmax}(QK^T / \sqrt{d_k})$ has shape `(batch, heads, seq_len, seq_len)`. Entry $W[b, h, i, j]$ represents how much position $i$ attends to position $j$.
+
+**Temporal range analysis:**
+- Average across heads: `W_avg = W.mean(dim=1)` — shape `(batch, seq_len, seq_len)`
+- If `W_avg[:, -1, -1] > 0.9` consistently, the agent is almost exclusively attending to the most recent frame
+
+**LSTM hidden state norm as attention proxy:**
+LSTM does not produce an explicit attention matrix, but the hidden state norm $\|h_t\|_2$ over time approximates the "importance" the network assigns to history. A flat norm curve suggests the LSTM gate is near saturation and not selectively reading history.
+
+## Edge Cases
+
+- **Vanishing gradients in deep networks:** Saliency computed at the input layer through many ReLU activations may be near-zero even for influential features. Prefer Grad-CAM on intermediate layers or use SmoothGrad (average over noisy copies of the input).
+- **Discrete action spaces:** Grad-CAM w.r.t. `argmax` is undefined (not differentiable). Use the pre-softmax logit of the chosen action instead.
+- **Normalization sensitivity:** Always normalize the saliency map to `[0, 1]` before overlaying. Raw gradient magnitudes vary across observations by orders of magnitude.

package/skills/blueprinting-system-architecture/SKILL.md

@@ -0,0 +1,95 @@
+---
+name: Blueprint System Architecture
+description: This skill should be used when the user asks to "blueprint the system architecture", "map the RL data flow", "design the RL pipeline", "define environment-agent interaction", "architect replay buffer placement", "design rollout and learner layers", or "draw the end-to-end RL system diagram". Do NOT hallucinate parameters outside the boundaries of Blueprint System Architecture.
+version: 0.1.0
+---
+
+# Blueprint System Architecture
+
+Map the end-to-end data flow between Environment, Core RL Loop, and Replay memories. This skill produces a structured architectural diagram and component breakdown for any Reinforcement Learning system, covering process topology, hardware placement, and data flow direction.
+
+## Core Components to Define
+
+Every RL system architecture consists of four layers that must be explicitly specified:
+
+| Layer | Role | Key Questions |
+|---|---|---|
+| **Rollout Worker Layer** | Generates experience | Async or sync? SubprocVecEnv or single process? |
+| **Experience Buffer Storage Layer** | Stores transitions | RAM-local or distributed (e.g. Ape-X)? |
+| **Learner / Optimizer Layer** | Computes gradients | CPU→GPU batch transfer or GPU-native? |
+| **Evaluation / Logging Hook Layer** | Tracks metrics | Frequency, checkpointing, W&B/TensorBoard? |
+
+## Step 1 — Map Component Interactivity
+
+Define the process topology first:
+
+- **Single-process**: Environment runs in the same process as the agent. Simple, debuggable. Use for prototyping.
+- **SubprocVecEnv**: Multiple environment instances run as separate OS processes. Use when environment computation is the bottleneck.
+- **Distributed (Ape-X / IMPALA style)**: Rollout workers run on separate machines and push experience to a central Replay Buffer. Learner pulls batches remotely. Use when scale > 1 machine.
+
+Specify the communication mechanism between components: shared memory, pipes, sockets, or gRPC.
+
+## Step 2 — Define Hardware Data Placement
+
+Data placement critically impacts throughput. Apply the following rules:
+
+**Standard CPU/GPU split (most systems):**
+1. Environment `step()` executes on CPU.
+2. Experience `(s, a, r, s', done)` stored in CPU RAM as NumPy arrays.
+3. Sampled mini-batches transferred to GPU via `.to(device)` only during gradient computation.
+4. Policy inference during rollout: run on CPU (single sample, low batch size) or GPU (vectorized envs).
+
+**GPU-native physics simulation (Isaac Gym / Brax):**
+- Environment state tensors live permanently on GPU memory.
+- Policy inference and gradient computation both occur on GPU.
+- **Critical rule:** Never call `.cpu()` or `.numpy()` on these tensors inside the rollout loop — this triggers a GPU→CPU bus transfer that destroys throughput. Keep tensors on-device until logging.
+
+## Step 3 — Specify Replay Buffer Architecture
+
+| System Type | Buffer Location | Buffer Class |
+|---|---|---|
+| On-Policy (PPO, A2C) | In-process RAM | `RolloutBuffer` (cleared after each update) |
+| Off-Policy single machine (SAC, DQN) | In-process RAM | `ReplayBuffer` (circular, persists across updates) |
+| Distributed Off-Policy (Ape-X) | Centralized Redis / Ray store | `PrioritizedReplayBuffer` (remote) |
+
+## Output Format
+
+Generate an ASCII diagram specifying all four layers, then follow with a structured breakdown:
+
+```
+┌─────────────────────────────────────────────┐
+│           ROLLOUT WORKER LAYER               │
+│  [Env 0] [Env 1] ... [Env N]  (CPU/GPU)     │
+│  Generates: (s, a, r, s', done) tuples       │
+└────────────────────┬────────────────────────┘
+                     │ push experience
+┌────────────────────▼────────────────────────┐
+│        EXPERIENCE BUFFER STORAGE LAYER       │
+│  ReplayBuffer / RolloutBuffer (RAM / Remote) │
+│  Capacity: N transitions                     │
+└────────────────────┬────────────────────────┘
+                     │ sample mini-batch → GPU
+┌────────────────────▼────────────────────────┐
+│          LEARNER / OPTIMIZER LAYER           │
+│  Actor + Critic forward pass (GPU)           │
+│  Loss computation + backprop (GPU)           │
+│  Parameter sync back to workers              │
+└────────────────────┬────────────────────────┘
+                     │ metrics
+┌────────────────────▼────────────────────────┐
+│       EVALUATION / LOGGING HOOK LAYER        │
+│  Episode return, entropy, loss, FPS          │
+│  Checkpoint saves, W&B / TensorBoard         │
+└─────────────────────────────────────────────┘
+```
+
+## Additional Resources
+
+### Reference Files
+
+- **`references/distributed-patterns.md`** — Ape-X, IMPALA, and R2D2 architectural patterns
+- **`references/hardware-placement-rules.md`** — GPU-native simulation rules (Isaac Gym / Brax)
+
+### Scripts
+
+- **`scripts/validate_architecture.py`** — Validates that a proposed architecture specification dict contains all required layers and fields

package/skills/blueprinting-system-architecture/references/distributed-patterns.md

@@ -0,0 +1,64 @@
+# Distributed RL Architecture Patterns
+
+## Ape-X (Distributed Prioritized Experience Replay)
+
+```
+Architecture:
+  N Actor workers (CPU) — each runs a copy of the policy
+  1 Central Replay Buffer (Redis / Ray store) — receives prioritized transitions
+  1 Learner (GPU) — pulls mini-batches, computes gradients, broadcasts weights back
+
+Data flow:
+  Actor → (s, a, r, s', done, priority) → Replay Buffer
+  Learner → sample(batch) → Replay Buffer
+  Learner → updated weights → Actors (async broadcast)
+```
+
+Key properties:
+- Actors run old policy parameters (lag of 1–100 gradient steps)
+- Prioritization is done at the actor level using local TD-error estimates
+- Learner runs at maximum throughput, decoupled from rollout speed
+- Scale: 360 actors in original DeepMind paper, scales linearly
+
+## IMPALA (Importance Weighted Actor-Learner Architecture)
+
+```
+Architecture:
+  N Actor workers — generate trajectories, send to queue
+  1 Learner — pulls trajectory chunks from queue, applies V-trace correction
+
+V-trace correction:
+  Corrects for off-policy data via importance sampling ratio clipping:
+  ρ_t = min(ρ̄, π_target(a_t|s_t) / π_behavior(a_t|s_t))
+```
+
+Key properties:
+- Designed for CPU-only actors with GPU learner
+- V-trace makes it robust to large policy lags (stable despite stale actors)
+- Policy gradient-based (not Q-learning), so compatible with continuous actions
+
+## R2D2 (Recurrent Replay Distributed DQN)
+
+```
+Architecture:
+  Like Ape-X but:
+  - Actor/Learner networks use LSTM for POMDPs
+  - Replay Buffer stores sequences (not individual transitions)
+  - Stored hidden state h_t alongside (s, a, r, done) sequences
+
+Burn-in: First K steps of each stored sequence are used to warm up LSTM state
+```
+
+Use R2D2 when: environment requires memory (POMDP) + distributed scale needed.
+
+## Single-Machine Multi-GPU
+
+For smaller scale (8–32 environments, 1–4 GPUs):
+```
+Process layout:
+  Main process: Learner on GPU 0
+  Subprocess pool: N env workers on CPU
+  Optional: Separate GPU for inference during rollout (if batch size > 256)
+```
+
+Recommended tools: `torch.multiprocessing`, `ray.remote`, or `stable-baselines3` VecEnv.