devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/storage.py

@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import fcntl
+from pathlib import Path
+from typing import Any, Callable, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class StorageBackend(Protocol):
+    def read_bytes(self, key: str) -> bytes: ...
+    def write_bytes(self, key: str, data: bytes) -> None: ...
+    def append_bytes_atomic(self, key: str, data: bytes) -> None: ...
+    def read_and_append_atomic(self, key: str, transform: Callable[[bytes], bytes]) -> None: ...
+    def exists(self, key: str) -> bool: ...
+    def list_keys(self, prefix: str) -> list[str]: ...
+    def delete(self, key: str) -> None: ...
+
+
+class LocalFileBackend:
+    """StorageBackend backed by the local filesystem. Thread-safe via fcntl.flock."""
+
+    def __init__(self, root: Path) -> None:
+        self._root = root
+
+    def _path(self, key: str) -> Path:
+        p = (self._root / key).resolve()
+        if not str(p).startswith(str(self._root.resolve())):
+            raise ValueError(f"Key escapes root: {key!r}")
+        return p
+
+    def read_bytes(self, key: str) -> bytes:
+        return self._path(key).read_bytes()
+
+    def write_bytes(self, key: str, data: bytes) -> None:
+        p = self._path(key)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        tmp = p.with_suffix(p.suffix + ".tmp")
+        tmp.write_bytes(data)
+        tmp.replace(p)
+
+    def append_bytes_atomic(self, key: str, data: bytes) -> None:
+        p = self._path(key)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        with p.open("ab") as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            try:
+                f.write(data)
+                f.flush()
+            finally:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
+    def read_and_append_atomic(self, key: str, transform: Callable[[bytes], bytes]) -> None:
+        """Read current content, call transform(content) to get bytes to append, write atomically."""
+        p = self._path(key)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        with p.open("a+b") as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            try:
+                f.seek(0)
+                current = f.read()
+                new_data = transform(current)
+                f.seek(0, 2)
+                f.write(new_data)
+                f.flush()
+            finally:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
+    def exists(self, key: str) -> bool:
+        return self._path(key).exists()
+
+    def list_keys(self, prefix: str) -> list[str]:
+        base = self._root / prefix
+        if not base.exists():
+            return []
+        return sorted(
+            str(p.relative_to(self._root))
+            for p in base.rglob("*")
+            if p.is_file()
+        )
+
+    def delete(self, key: str) -> None:
+        self._path(key).unlink(missing_ok=True)
+
+
+class R2Backend:
+    """StorageBackend backed by Cloudflare R2 (S3-compatible). Uses conditional PUTs for atomic append."""
+
+    def __init__(self, client: Any, bucket: str, prefix: str) -> None:
+        # prefix = "{org_id}/{repo_id}/.GCC"  (no trailing slash)
+        self._s3 = client
+        self._bucket = bucket
+        self._prefix = prefix.rstrip("/")
+
+    def _key(self, relative: str) -> str:
+        return f"{self._prefix}/{relative}"
+
+    def read_bytes(self, key: str) -> bytes:
+        try:
+            resp = self._s3.get_object(Bucket=self._bucket, Key=self._key(key))
+            return resp["Body"].read()
+        except self._s3.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] in ("NoSuchKey", "404"):
+                raise FileNotFoundError(key)
+            raise
+
+    def write_bytes(self, key: str, data: bytes) -> None:
+        self._s3.put_object(Bucket=self._bucket, Key=self._key(key), Body=data)
+
+    def append_bytes_atomic(self, key: str, data: bytes) -> None:
+        self.read_and_append_atomic(key, lambda current: data)
+
+    def read_and_append_atomic(self, key: str, transform: Callable[[bytes], bytes]) -> None:
+        full_key = self._key(key)
+        for attempt in range(10):
+            try:
+                resp = self._s3.get_object(Bucket=self._bucket, Key=full_key)
+                current = resp["Body"].read()
+                etag = resp["ETag"].strip('"')
+            except self._s3.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] in ("NoSuchKey", "404"):
+                    current, etag = b"", None
+                else:
+                    raise
+
+            new_data = transform(current)
+            new_content = current + new_data
+
+            try:
+                kwargs: dict = {"Bucket": self._bucket, "Key": full_key, "Body": new_content}
+                if etag:
+                    kwargs["IfMatch"] = f'"{etag}"'
+                else:
+                    # Blob did not exist — use If-None-Match: * to guard concurrent creates.
+                    kwargs["IfNoneMatch"] = "*"
+                self._s3.put_object(**kwargs)
+                return
+            except self._s3.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] in ("PreconditionFailed", "ConditionalRequestConflict"):
+                    continue
+                raise
+        raise RuntimeError(f"R2Backend: failed to atomically update {key} after 10 retries")
+
+    def exists(self, key: str) -> bool:
+        try:
+            self.read_bytes(key)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def list_keys(self, prefix: str) -> list[str]:
+        full_prefix = self._key(prefix)
+        resp = self._s3.list_objects_v2(Bucket=self._bucket, Prefix=full_prefix)
+        contents = resp.get("Contents", [])
+        strip = self._prefix + "/"
+        return sorted(item["Key"].removeprefix(strip) for item in contents)
+
+    def delete(self, key: str) -> None:
+        self._s3.delete_object(Bucket=self._bucket, Key=self._key(key))
+
+
+class AzureBlobBackend:
+    """StorageBackend backed by Azure Blob Storage. Uses ETag conditional PUTs for atomic append."""
+
+    def __init__(self, client: Any, container: str, prefix: str) -> None:
+        # prefix = "{org_id}/{repo_id}/.GCC"  (no trailing slash)
+        self._service = client
+        self._container = container
+        self._prefix = prefix.rstrip("/")
+        self._container_client = client.get_container_client(container)
+
+    def _blob_name(self, relative: str) -> str:
+        return f"{self._prefix}/{relative}" if self._prefix else relative
+
+    def read_bytes(self, key: str) -> bytes:
+        from azure.core.exceptions import ResourceNotFoundError
+        try:
+            blob_client = self._container_client.get_blob_client(self._blob_name(key))
+            downloader = blob_client.download_blob()
+            return downloader.readall()
+        except ResourceNotFoundError as exc:
+            raise FileNotFoundError(key) from exc
+
+    def write_bytes(self, key: str, data: bytes) -> None:
+        blob_client = self._container_client.get_blob_client(self._blob_name(key))
+        blob_client.upload_blob(data, overwrite=True)
+
+    def append_bytes_atomic(self, key: str, data: bytes) -> None:
+        self.read_and_append_atomic(key, lambda _current: data)
+
+    def read_and_append_atomic(self, key: str, transform: Callable[[bytes], bytes]) -> None:
+        from azure.core.exceptions import ResourceExistsError, ResourceModifiedError, ResourceNotFoundError
+
+        try:
+            from azure.core import MatchConditions
+        except ImportError:
+            from azure.core.conditions import MatchConditions  # type: ignore[no-redef]
+        blob_client = self._container_client.get_blob_client(self._blob_name(key))
+
+        for _attempt in range(10):
+            try:
+                downloader = blob_client.download_blob()
+                current = downloader.readall()
+                etag = downloader.properties["etag"].strip('"')
+            except ResourceNotFoundError:
+                current, etag = b"", None
+
+            new_data = transform(current)
+            new_content = current + new_data
+
+            try:
+                if etag:
+                    blob_client.upload_blob(
+                        new_content,
+                        overwrite=True,
+                        etag=etag,
+                        match_condition=MatchConditions.IfNotModified,
+                    )
+                else:
+                    # Blob did not exist — use overwrite=False to guard concurrent creates.
+                    blob_client.upload_blob(new_content, overwrite=False)
+                return
+            except (ResourceModifiedError, ResourceExistsError):
+                continue
+        raise RuntimeError(f"AzureBlobBackend: failed to atomically update {key} after 10 retries")
+
+    def exists(self, key: str) -> bool:
+        from azure.core.exceptions import ResourceNotFoundError
+        try:
+            self._container_client.get_blob_client(self._blob_name(key)).get_blob_properties()
+            return True
+        except ResourceNotFoundError:
+            return False
+
+    def list_keys(self, prefix: str) -> list[str]:
+        if self._prefix:
+            full_prefix = f"{self._prefix}/{prefix}" if prefix else self._prefix + "/"
+        else:
+            full_prefix = prefix
+        blobs = self._container_client.list_blobs(name_starts_with=full_prefix)
+        strip = self._prefix + "/" if self._prefix else ""
+        return sorted(b.name.removeprefix(strip) for b in blobs)
+
+    def delete(self, key: str) -> None:
+        from azure.core.exceptions import ResourceNotFoundError
+        blob_client = self._container_client.get_blob_client(self._blob_name(key))
+        try:
+            blob_client.delete_blob()
+        except ResourceNotFoundError:
+            pass
+
+
+def create_backend_from_env(prefix: str = "") -> "StorageBackend":
+    """
+    Create a StorageBackend from environment variables.
+
+    DEVTORCH_STORAGE controls which backend to use:
+      r2     — Cloudflare R2 (boto3, requires R2_ACCOUNT_ID, R2_BUCKET, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY)
+      s3     — Native AWS S3 (boto3, requires AWS_S3_BUCKET; uses IAM/env auth automatically)
+      azure  — Azure Blob Storage (requires DEVTORCH_AZURE_CONTAINER + AZURE_STORAGE_CONNECTION_STRING
+               or AZURE_STORAGE_ACCOUNT_URL)
+      local  — Local filesystem (uses DEVTORCH_GCC_PATH or current directory)
+
+    prefix is prepended to all storage keys (e.g. "{org_id}" for auth backends).
+    """
+    import os
+
+    storage = os.environ.get("DEVTORCH_STORAGE", "r2").lower()
+
+    if storage == "r2":
+        import boto3
+        bucket = os.environ.get("DEVTORCH_BUCKET") or os.environ["R2_BUCKET"]
+        account_id = os.environ["R2_ACCOUNT_ID"]
+        access_key = os.environ.get("DEVTORCH_ACCESS_KEY_ID") or os.environ["R2_ACCESS_KEY_ID"]
+        secret_key = os.environ.get("DEVTORCH_SECRET_ACCESS_KEY") or os.environ["R2_SECRET_ACCESS_KEY"]
+        s3 = boto3.client(
+            "s3",
+            endpoint_url=f"https://{account_id}.r2.cloudflarestorage.com",
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            region_name="auto",
+        )
+        return R2Backend(client=s3, bucket=bucket, prefix=prefix)
+
+    elif storage == "s3":
+        import boto3
+        bucket = os.environ.get("DEVTORCH_BUCKET") or os.environ["AWS_S3_BUCKET"]
+        s3 = boto3.client("s3")  # uses IAM role, env vars, or ~/.aws/credentials
+        return R2Backend(client=s3, bucket=bucket, prefix=prefix)
+
+    elif storage == "azure":
+        from azure.storage.blob import BlobServiceClient
+        container = os.environ["DEVTORCH_AZURE_CONTAINER"]
+        conn_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
+        if conn_str:
+            service = BlobServiceClient.from_connection_string(conn_str)
+        else:
+            account_url = os.environ["AZURE_STORAGE_ACCOUNT_URL"]
+            service = BlobServiceClient(account_url=account_url)
+        return AzureBlobBackend(client=service, container=container, prefix=prefix)
+
+    elif storage == "local":
+        gcc_path = Path(os.environ.get("DEVTORCH_GCC_PATH", "."))
+        return LocalFileBackend(gcc_path)
+
+    else:
+        raise ValueError(
+            f"Unknown DEVTORCH_STORAGE: {storage!r}. Must be one of: r2, s3, azure, local."
+        )

devtorch_core/templates/__init__.py

@@ -0,0 +1,6 @@
+"""Project templates for `devtorch init --template`."""
+from __future__ import annotations
+
+from devtorch_core.templates.engine import apply_template, cmd_init_template, list_templates
+
+__all__ = ["apply_template", "cmd_init_template", "list_templates"]

devtorch_core/templates/engine.py

@@ -0,0 +1,122 @@
+"""Built-in project templates for `devtorch init --template`."""
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from devtorch_core import GCCRepository
+
+
+@dataclass
+class Template:
+    """A project template defines default concepts and a governance note."""
+
+    name: str
+    display_name: str
+    project_type: str
+    concepts: list[tuple[str, str]]
+    governance_note: str
+
+
+# Built-in template registry keyed by template name.
+_TEMPLATES: dict[str, Template] = {}
+
+
+def register_template(template: Template) -> None:
+    """Register a built-in or custom template."""
+    _TEMPLATES[template.name] = template
+
+
+def _load_builtin_templates() -> None:
+    """Lazily import and register built-in templates."""
+    if _TEMPLATES:
+        return
+    from devtorch_core.templates import python, typescript, react, go, infra
+
+    for module in (python, typescript, react, go, infra):
+        register_template(module.TEMPLATE)
+
+
+def list_templates() -> list[dict[str, Any]]:
+    """Return a list of available templates with metadata."""
+    _load_builtin_templates()
+    return [
+        {
+            "name": t.name,
+            "display_name": t.display_name,
+            "project_type": t.project_type,
+        }
+        for t in _TEMPLATES.values()
+    ]
+
+
+def _get_template(name: str) -> Template:
+    """Look up a template by name; raise ValueError if unknown."""
+    _load_builtin_templates()
+    try:
+        return _TEMPLATES[name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown template: {name}") from exc
+
+
+def apply_template(
+    gcc_dir: Path | str,
+    template_name: str,
+    project_type: str | None = None,
+) -> None:
+    """
+    Apply a project template to a GCC repository.
+
+    The repository at `gcc_dir` is initialized if it has not been already.
+    Default concepts are written to `.GCC/concepts/` and the template's
+    governance note is persisted to `.GCC/templates/policy.md`.
+    """
+    template = _get_template(template_name)
+    repo = GCCRepository.at(Path(gcc_dir).resolve())
+
+    if not repo.is_initialized():
+        repo.init()
+
+    for concept_name, definition in template.concepts:
+        repo.concept_add(concept_name, definition)
+
+    policy_dir = repo.gcc_dir / "templates"
+    policy_dir.mkdir(parents=True, exist_ok=True)
+    policy_path = policy_dir / "policy.md"
+
+    effective_type = project_type or template.project_type
+    note = f"""# Template: {template.display_name}
+
+Project type: {effective_type}
+Default governance focus: {template.governance_note}
+
+Applied concepts: {', '.join(name for name, _ in template.concepts)}
+"""
+    policy_path.write_text(note, encoding="utf-8")
+
+
+def cmd_init_template(args: Any) -> int:
+    """
+    Helper for CLI integration with `devtorch init --template <type>`.
+
+    `args` is a Namespace-like object with at least `path` and `template`.
+    Optional `project_type` customizes the template's declared project type.
+    """
+    template_name = getattr(args, "template", None)
+    if not template_name:
+        print("error: --template is required", file=sys.stderr)
+        return 1
+
+    project_root = Path(getattr(args, "path", ".")).resolve()
+    project_type = getattr(args, "project_type", None)
+
+    try:
+        apply_template(project_root, template_name, project_type)
+    except ValueError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 1
+
+    print(f"Applied template '{template_name}' to {project_root}")
+    return 0

devtorch_core/templates/go.py

@@ -0,0 +1,18 @@
+"""Built-in Go project template."""
+from __future__ import annotations
+
+from devtorch_core.templates.engine import Template
+
+TEMPLATE = Template(
+    name="go",
+    display_name="Go",
+    project_type="go",
+    concepts=[
+        ("auth", "Authentication, authorization, and identity management."),
+        ("schema", "Structs, validation, and serialization contracts."),
+        ("api", "HTTP handlers, gRPC services, and public interfaces."),
+        ("data", "Storage, repositories, and data access patterns."),
+        ("cmd", "Command-line entry points and service binaries."),
+    ],
+    governance_note="Go projects default to explicit interfaces, small command entry points, and clear separation between API handlers, business logic, and data repositories.",
+)

devtorch_core/templates/infra.py

@@ -0,0 +1,19 @@
+"""Built-in infrastructure project template."""
+from __future__ import annotations
+
+from devtorch_core.templates.engine import Template
+
+TEMPLATE = Template(
+    name="infra",
+    display_name="Infrastructure",
+    project_type="infra",
+    concepts=[
+        ("auth", "Identity, access control, and least-privilege policies."),
+        ("schema", "Resource definitions, configuration schemas, and versioning."),
+        ("api", "Gateway, load balancer, and service mesh configuration."),
+        ("data", "Databases, object storage, and backup policies."),
+        ("networking", "VPCs, subnets, DNS, and firewall rules."),
+        ("secrets", "Secret management, rotation, and encryption at rest."),
+    ],
+    governance_note="Infrastructure projects default to declarative resource definitions, least-privilege access, explicit networking boundaries, and auditable secret management.",
+)

devtorch_core/templates/library/__init__.py

@@ -0,0 +1,18 @@
+"""Reasoning template library package."""
+from __future__ import annotations
+
+from devtorch_core.templates.library.engine import (
+    apply_template,
+    discover_custom_templates,
+    list_builtin_templates,
+    load_builtin_template,
+    render_template,
+)
+
+__all__ = [
+    "apply_template",
+    "discover_custom_templates",
+    "list_builtin_templates",
+    "load_builtin_template",
+    "render_template",
+]

devtorch_core/templates/library/api_design.md

@@ -0,0 +1,27 @@
+---
+name: api_design
+description: Design rationale for an API or interface change
+tags: [reasoning, api, design]
+---
+# API Design: {{title}}
+
+## Goal
+{{goal}}
+
+## Constraints
+{{constraints}}
+
+## Proposed interface
+{{interface}}
+
+## Design rationale
+{{reasoning}}
+
+## Alternatives considered
+{{alternatives}}
+
+## Risks
+{{risks}}
+
+## Migration plan
+{{migration}}

devtorch_core/templates/library/bug_fix.md

@@ -0,0 +1,27 @@
+---
+name: bug_fix
+description: Capture the investigation, root cause, and fix for a bug
+tags: [reasoning, bug, debugging]
+---
+# Bug Fix: {{title}}
+
+## Observed behaviour
+{{observed}}
+
+## Expected behaviour
+{{expected}}
+
+## Reproduction steps
+{{repro_steps}}
+
+## Root cause analysis
+{{reasoning}}
+
+## Fix
+{{fix}}
+
+## Verification
+{{verification}}
+
+## Prevention
+{{prevention}}

devtorch_core/templates/library/decision_record.md

@@ -0,0 +1,27 @@
+---
+name: decision_record
+description: Structured reasoning record for architectural or product decisions
+tags: [reasoning, decision, architecture]
+---
+# Decision Record: {{title}}
+
+## Context
+{{context}}
+
+## Problem statement
+{{problem}}
+
+## Options considered
+{{options}}
+
+## Decision
+{{decision}}
+
+## Rationale
+{{reasoning}}
+
+## Consequences
+{{consequences}}
+
+## Related concepts
+{{related_concepts}}