microcoreos 0.1.0__py3-none-any.whl

microcoreos/_template/dev_infra/cache_probe.py

@@ -0,0 +1,176 @@
+"""Cache probe — proves whether an inference engine actually reuses the shared
+prompt prefix (the "canonical executor prompt" of docs/PARALLEL_DEVELOPMENT.md).
+
+It sends three requests and compares how many prompt tokens the engine REALLY
+processed (not how many you sent):
+
+  1. COLD    shared prefix + task A          -> baseline (everything processed)
+  2. WARM    identical prefix + task B       -> should process ~task tokens only
+  3. BROKEN  prefix with 1 byte changed + B  -> back to baseline (negative control)
+
+PASS = warm processed-tokens << cold, and broken ~= cold.
+
+The shared prefix is your real workload: AI_CONTEXT.md + plans/active_plan.yaml.
+
+Usage:
+  python dev_infra/cache_probe.py --backend ollama    --url http://localhost:11434 --model llama3.1
+  python dev_infra/cache_probe.py --backend llamacpp  --url http://localhost:8080
+  python dev_infra/cache_probe.py --backend openai    --url http://localhost:8000 --model my-model   # vLLM etc.
+  python dev_infra/cache_probe.py --backend anthropic --model claude-haiku-4-5     # needs ANTHROPIC_API_KEY
+
+No dependencies beyond the standard library.
+"""
+import argparse
+import json
+import os
+import time
+import urllib.request
+
+TASK_A = "Implement feature CreateThingPlugin from the plan above. Reply with only the file path you would create."
+TASK_B = "Implement feature DeleteThingPlugin from the plan above. Reply with only the file path you would create."
+
+
+def read_shared_prefix() -> str:
+    parts = []
+    for path in ("AI_CONTEXT.md", "plans/active_plan.yaml"):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                parts.append(f"===== {path} =====\n{f.read()}")
+        except FileNotFoundError:
+            parts.append(f"===== {path} ===== (missing)")
+    parts.append("===== RULES =====\nOne file = one feature. Follow the plan exactly.")
+    return "\n\n".join(parts)
+
+
+def post_json(url: str, body: dict, headers: dict) -> dict:
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", **headers},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=600) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+# ── Backends ────────────────────────────────────────────────────────────────
+# Each returns {"processed": int, "cached": int|None, "wall_s": float}
+# "processed" = prompt tokens the engine actually evaluated this request.
+
+def probe_ollama(url, model, prefix, task):
+    t0 = time.monotonic()
+    r = post_json(f"{url}/api/chat", {
+        "model": model, "stream": False,
+        "messages": [{"role": "user", "content": prefix + "\n\n" + task}],
+        "options": {"num_predict": 32},
+    }, {})
+    return {"processed": r.get("prompt_eval_count", -1), "cached": None,
+            "wall_s": time.monotonic() - t0}
+
+
+def probe_llamacpp(url, model, prefix, task):
+    t0 = time.monotonic()
+    r = post_json(f"{url}/completion", {
+        "prompt": prefix + "\n\n" + task,
+        "n_predict": 32, "cache_prompt": True,
+    }, {})
+    timings = r.get("timings", {})
+    return {"processed": timings.get("prompt_n", -1), "cached": None,
+            "wall_s": time.monotonic() - t0}
+
+
+def probe_openai(url, model, prefix, task):
+    """OpenAI-compatible servers (vLLM with prefix caching, OpenAI itself)."""
+    t0 = time.monotonic()
+    headers = {}
+    if os.getenv("OPENAI_API_KEY"):
+        headers["Authorization"] = f"Bearer {os.environ['OPENAI_API_KEY']}"
+    r = post_json(f"{url}/v1/chat/completions", {
+        "model": model, "max_tokens": 32,
+        "messages": [{"role": "user", "content": prefix + "\n\n" + task}],
+    }, headers)
+    usage = r.get("usage", {})
+    total = usage.get("prompt_tokens", -1)
+    cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0)
+    return {"processed": total - cached, "cached": cached,
+            "wall_s": time.monotonic() - t0}
+
+
+def _anthropic_headers():
+    h = {"anthropic-version": "2023-06-01"}
+    if os.getenv("ANTHROPIC_API_KEY"):
+        h["x-api-key"] = os.environ["ANTHROPIC_API_KEY"]
+    elif os.getenv("ANTHROPIC_AUTH_TOKEN"):  # OAuth token (e.g. from ant auth / Claude Code)
+        h["Authorization"] = f"Bearer {os.environ['ANTHROPIC_AUTH_TOKEN']}"
+        h["anthropic-beta"] = "oauth-2025-04-20"
+    else:
+        raise SystemExit("Set ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN")
+    return h
+
+
+def probe_anthropic(url, model, prefix, task):
+    t0 = time.monotonic()
+    r = post_json(f"{url}/v1/messages", {
+        "model": model, "max_tokens": 32,
+        "system": [{"type": "text", "text": prefix,
+                    "cache_control": {"type": "ephemeral"}}],
+        "messages": [{"role": "user", "content": task}],
+    }, _anthropic_headers())
+    u = r.get("usage", {})
+    processed = u.get("input_tokens", -1) + u.get("cache_creation_input_tokens", 0)
+    return {"processed": processed, "cached": u.get("cache_read_input_tokens", 0),
+            "wall_s": time.monotonic() - t0}
+
+
+BACKENDS = {
+    "ollama": (probe_ollama, "http://localhost:11434"),
+    "llamacpp": (probe_llamacpp, "http://localhost:8080"),
+    "openai": (probe_openai, "http://localhost:8000"),
+    "anthropic": (probe_anthropic, "https://api.anthropic.com"),
+}
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--backend", choices=BACKENDS, required=True)
+    ap.add_argument("--model", default="")
+    ap.add_argument("--url", default="")
+    args = ap.parse_args()
+
+    probe, default_url = BACKENDS[args.backend]
+    url = (args.url or default_url).rstrip("/")
+    prefix = read_shared_prefix()
+    # Negative control: one byte changed at position 0 breaks the whole prefix.
+    broken_prefix = "!" + prefix[1:]
+
+    runs = [
+        ("COLD   (prefix + task A)", prefix, TASK_A),
+        ("WARM   (same prefix + task B)", prefix, TASK_B),
+        ("BROKEN (1 byte changed + task B)", broken_prefix, TASK_B),
+    ]
+    results = []
+    print(f"backend={args.backend}  model={args.model or '(default)'}  url={url}")
+    print(f"shared prefix: {len(prefix)} chars (~{len(prefix) // 4} tokens)\n")
+    for name, pfx, task in runs:
+        res = probe(url, args.model, pfx, task)
+        results.append(res)
+        cached = f"  cached={res['cached']}" if res["cached"] is not None else ""
+        print(f"{name:38} processed={res['processed']:>7}{cached}  wall={res['wall_s']:.2f}s")
+
+    cold, warm, broken = (r["processed"] for r in results)
+    print()
+    if 0 <= warm < cold * 0.5 and broken > cold * 0.5:
+        print(f"PASS — WARM processed only {warm}/{cold} tokens (cache hit), "
+              f"and BROKEN reprocessed {broken} (prefix rule confirmed).")
+    elif 0 <= warm < cold * 0.5:
+        print(f"PARTIAL — WARM hit the cache ({warm}/{cold}), but BROKEN also "
+              f"avoided reprocessing ({broken}) — check the negative control.")
+    else:
+        print(f"FAIL — WARM reprocessed {warm}/{cold} tokens. No prefix reuse: "
+              f"check that prefix caching is enabled on the server, the prefix "
+              f"is byte-identical, and (hosted) the prefix meets the minimum "
+              f"cacheable size for the model.")
+
+
+if __name__ == "__main__":
+    main()

microcoreos/_template/dev_infra/docker-compose.yml

@@ -0,0 +1,137 @@
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# MicroCoreOS — Development Infrastructure
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+#
+# Uso:
+#   docker compose -f dev_infra/docker-compose.yml up -d
+#   docker compose -f dev_infra/docker-compose.yml down
+#
+# Los datos persisten en volúmenes Docker entre reinicios.
+# Para borrar todo y empezar limpio:
+#   docker compose -f dev_infra/docker-compose.yml down -v
+#
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+services:
+
+  # ─── PostgreSQL ───────────────────────────────────────────
+  postgres:
+    image: postgres:17-alpine
+    container_name: microcoreos_postgres
+    restart: unless-stopped
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: microcoreos
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── Redis ────────────────────────────────────────────────
+  redis:
+    image: redis:7-alpine
+    container_name: microcoreos_redis
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── RabbitMQ ─────────────────────────────────────────────
+  rabbitmq:
+    image: rabbitmq:3.13-alpine
+    container_name: microcoreos_rabbitmq
+    restart: unless-stopped
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+    volumes:
+      - rabbitmq_data:/var/lib/rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ─── Kafka (KRaft, single node) ───────────────────────────
+  kafka:
+    image: apache/kafka:3.8.0
+    container_name: microcoreos_kafka
+    restart: unless-stopped
+    ports:
+      - "9092:9092"
+    environment:
+      # Single-node KRaft: the container is broker AND controller.
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+    volumes:
+      - kafka_data:/var/lib/kafka/data
+    healthcheck:
+      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+
+  # ─── RustFS (S3-compatible object storage) ────────────────
+  rustfs:
+    image: rustfs/rustfs:latest
+    container_name: microcoreos_rustfs
+    restart: unless-stopped
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      RUSTFS_ACCESS_KEY: minioadmin
+      RUSTFS_SECRET_KEY: minioadmin
+    volumes:
+      - rustfs_data:/data
+    command: server /data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # ─── Jaeger (OpenTelemetry / OTLP tracing UI) ─────────────
+  # UI: http://localhost:16686
+  # App: en .env → OTEL_ENABLED=true y
+  #      OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+  # Requiere: uv add opentelemetry-sdk opentelemetry-exporter-otlp
+  # Almacenamiento en memoria (dev): las trazas se pierden al reiniciar.
+  jaeger:
+    image: jaegertracing/all-in-one:1.60
+    container_name: microcoreos_jaeger
+    restart: unless-stopped
+    ports:
+      - "16686:16686"   # Web UI
+      - "4317:4317"     # OTLP gRPC (el que usa el tool telemetry)
+      - "4318:4318"     # OTLP HTTP
+    environment:
+      COLLECTOR_OTLP_ENABLED: "true"
+
+volumes:
+  postgres_data:
+  redis_data:
+  rabbitmq_data:
+  kafka_data:
+  rustfs_data:

microcoreos/_template/docs/CLI.md

@@ -0,0 +1,187 @@
+# The `microcoreos` command
+
+Installed by the package. Five commands: create a project, install an extra,
+see what changed upstream, boot, boot with reload.
+
+```
+microcoreos new <path> [--force] [--no-ai-kit]   Materialize a new project
+microcoreos add <extra> [--no-install]           Install an extra completely
+microcoreos upgrade [--apply]                    Report/apply upstream changes
+microcoreos [run] [--boot-tool <tool>]           Boot the Kernel
+microcoreos dev                                  Boot with auto-reload
+```
+
+Except for `new`, run them from the root of a project — the directory holding
+`tools/`, `domains/` and `plans/`. Running elsewhere is refused rather than
+booting an empty system and calling it Ready.
+
+---
+
+## `microcoreos new <path>`
+
+Copies the framework's source into your directory: `tools/`, `domains/system`,
+`domains/devtools`, `extras/`, `plans/`, `dev_infra/`, `main.py`, `Dockerfile`,
+`.env.example` — plus the AI kit (`AGENTS.md`, `INSTRUCTIONS_FOR_AI.md`,
+`.agent/`, `docs/`).
+
+One list decides all of it: `scaffold.RUNTIME_ENTRIES`. The wheel's copy under
+`_template/` is generated from that same list by `hatch_build.py`, so what a
+checkout copies and what the package ships cannot disagree.
+
+**Why copied instead of imported.** Installing and swapping infrastructure here
+IS file placement (`mv extras/available_tools/postgresql tools/`, dropping a
+`{name}_driver.py` into `tools/event_bus/`). None of that works against
+`site-packages`: it may be read-only, and anything written there is wiped by
+the next upgrade. Only the Kernel — `microcoreos/` itself — stays in the
+package.
+
+It also writes, only when absent: `.env` (from `.env.example`),
+`pyproject.toml`, and a `README.md` for the project. An existing one is never
+overwritten, so `uv init && uv add microcoreos && microcoreos new .` works.
+
+| Flag | Effect |
+|---|---|
+| `--force` | Materialize even if `tools/` or `domains/` already exist |
+| `--no-ai-kit` | Skip `AGENTS.md`, `INSTRUCTIONS_FOR_AI.md`, `.agent/`, `docs/` |
+
+**Not materialized:** `ping` (the hello-world) and auth. A fresh project has no
+`tools/auth` and no `domains/users` — no users table, no JWT, and no
+`AUTH_SECRET_KEY` to set before it will boot. `http_server_tool` takes
+`auth_validator` as an optional callback and never imports auth, so nothing
+misses it.
+
+`microcoreos add auth` installs it: the tool, plus the four plugins that ARE
+auth — `create_user`, `login`, `get_me`, `logout` — with the model and the
+migration. The CRUD around them (list, get-by-id, update, delete) is not
+shipped: that is what you write for your own entities.
+
+---
+
+## `microcoreos add <extra>`
+
+Activating an extra is three acts, and skipping any one fails somewhere
+different:
+
+| Act | Skipping it |
+|---|---|
+| Install the optional dependency | `No module named 'asyncpg'` at boot |
+| Move the source into `tools/`/`domains/` | Nothing happens — discovery only sees those directories |
+| Add its settings to `.env` | Boots, then connects nowhere |
+
+This does all three:
+
+```bash
+microcoreos add postgres
+```
+
+```
+📦 Installing extra 'postgres'
+
+   $ uv add 'microcoreos[postgres]'
+   ✓ extras/available_tools/postgresql → tools/postgresql
+   ✓ .env += PG_HOST, PG_PORT, PG_USER, PG_PASSWORD, PG_DATABASE
+```
+
+Available: `auth`, `ping`, `postgres`, `redis`, `s3`, `scheduler`, `kafka`,
+`rabbitmq`, `chaos`. Run `microcoreos add` with no argument to list them.
+
+`auth` and `ping` are the two that need no external service. `auth` is an extra
+because a users table, a roles model and a JWT flavour are not something a
+framework should impose; `ping` is a single `GET /ping` you read for the shape
+of a minimal plugin and then delete.
+
+Three rules it follows:
+
+- **The tool moves before its domain.** A plugin cannot exist without the tool
+  it asks for, so the other order leaves a boot that aborts the plugin.
+- **Drivers are not tools.** `kafka` and `rabbitmq` do not become
+  `tools/kafka/`; the `*_driver.py` drops into `tools/event_bus/` and `.env`
+  gets `EVENT_BUS_DRIVER=kafka`.
+- **`.env` is never overwritten.** A value already there is your decision.
+  Re-running `add` is a no-op.
+
+`uv add` runs only when there is a `pyproject.toml` and `uv` is on PATH —
+otherwise the command is printed for you to run. `--no-install` skips that step.
+
+---
+
+## `microcoreos upgrade`
+
+The mitigation for honest vendoring: your tools and domains are your source, so
+a fix upstream does not reach them on its own.
+
+`new` records the SHA-256 of every file it writes in
+`.microcoreos/manifest.json`. That baseline is what makes three otherwise
+indistinguishable states distinguishable:
+
+| local | upstream | verdict |
+|---|---|---|
+| = baseline | changed | **safe to update** — you never touched it |
+| changed | = baseline | **yours** — left alone, always |
+| changed | changed | **conflict** — reported, never written |
+| = baseline | withdrawn | **deleted** — the framework's file to take back |
+| changed | withdrawn | **released** — kept, and no longer tracked at all |
+
+```bash
+microcoreos upgrade            # report only (default)
+microcoreos upgrade --apply    # write the safe ones, move the baseline
+```
+
+`--apply` never touches a conflict and never resurrects a file you deleted —
+removing a tool you do not use is a supported act, not damage.
+
+**A file dropped upstream is deleted here, unless you edited it.** The rule is
+the same one as everywhere else: untouched is the framework's to withdraw, and
+an empty `tools/<name>/` goes with the last file in it. If you did edit it, it
+stays and leaves the baseline for good — upstream no longer ships it and you
+changed it, so there is nothing left to compare, and re-reporting it on every
+run would just teach you to skip the output.
+
+If most of the baseline appears to have vanished, nothing is deleted: that is
+what a partial or broken install looks like, not what a release looks like.
+
+**The manifest belongs in version control.** The scaffolded `.gitignore` does
+not exclude it on purpose: a teammate who clones the project without it cannot
+upgrade, because their edits would be indistinguishable from stale files.
+Without a manifest the command refuses outright — it is written by
+`microcoreos new`, and its absence is an error rather than a degraded mode.
+
+**Extras you installed are tracked to where they moved.** `microcoreos add`
+records the move, and the baseline stays keyed by the file's upstream path —
+so a fix to `extras/available_tools/scheduler/` reaches `tools/scheduler/`
+where you actually put it.
+
+Moving a folder yourself works too, including to a name no convention could
+guess (`mv extras/available_tools/postgresql tools/my-db`). Names cannot follow
+that, so content does: an unedited file still hashes to its baseline digest
+wherever it now sits, and the move is written into the manifest the first time
+`upgrade` runs — after which the folder is tracked by name and you can edit it
+freely. A digest matching more than one candidate proves nothing and is
+ignored, so nothing is claimed on a guess.
+
+---
+
+## `microcoreos run` / `microcoreos dev`
+
+`microcoreos` alone is `microcoreos run`. Identical to `uv run main.py` —
+the root `main.py` is a shim over this same code path, so both are the same
+boot.
+
+```bash
+microcoreos                          # boot
+microcoreos run --boot-tool db       # boot ONE tool in isolation, then exit
+microcoreos dev                      # boot, reload on .py changes
+```
+
+`--boot-tool` is the migrations pipeline entry point:
+`DB_AUTO_MIGRATE=true microcoreos run --boot-tool db` — see
+[ELASTIC_DEPLOYMENT.md](ELASTIC_DEPLOYMENT.md).
+
+`dev` needs `watchfiles` (`uv add --dev watchfiles`); it says so if missing.
+
+**Two things the command does that `python main.py` gets for free.** A console
+script starts with the venv's `bin/` as `sys.path[0]`, not your project, so the
+project root is inserted explicitly — without it every `import tools.*` fails
+and the Kernel reports an empty system as Ready. And `.env` is loaded by
+explicit path, because bare `load_dotenv()` searches upward from its *caller*,
+which installed is `site-packages`, not your project.