modelable 1.0.0__py3-none-any.whl

modelable/llm/provenance.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class WriteProvenance:
+    command: str
+    artifact_path: str
+    provider: str
+    model: str
+    validation_status: str
+    diagnostics_repaired: int
+    inputs: dict[str, str]
+
+    def as_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def build_write_provenance(
+    *,
+    command: str,
+    artifact_path: Path,
+    provider: str,
+    model: str,
+    validation_status: str,
+    diagnostics_repaired: int,
+    inputs: dict[str, str],
+) -> WriteProvenance:
+    return WriteProvenance(
+        command=command,
+        artifact_path=str(artifact_path),
+        provider=provider,
+        model=model,
+        validation_status=validation_status,
+        diagnostics_repaired=diagnostics_repaired,
+        inputs=inputs,
+    )
+
+
+def provenance_sidecar_path(artifact_path: Path) -> Path:
+    return artifact_path.with_name(f"{artifact_path.name}.provenance.json")
+
+
+def render_write_provenance(provenance: WriteProvenance) -> str:
+    return json.dumps(provenance.as_dict(), indent=2, ensure_ascii=False, sort_keys=True) + "\n"
+
+
+def write_provenance_sidecar(artifact_path: Path, provenance: WriteProvenance) -> Path:
+    sidecar_path = provenance_sidecar_path(artifact_path)
+    sidecar_path.write_text(render_write_provenance(provenance), encoding="utf-8")
+    return sidecar_path
+
+
+@dataclass(frozen=True)
+class AttachmentRecord:
+    ref: str
+    source_format: str
+    source_name: str
+    source_path: str
+    source_hash: str
+    from_version: int
+    to_version: int | None
+    change_kind: str | None
+    changes: list[dict[str, object]]
+
+    def as_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def attachment_sidecar_path(artifact_path: Path) -> Path:
+    return artifact_path.with_name(f"{artifact_path.name}.attachments.json")
+
+
+def write_attachment_record(artifact_path: Path, record: AttachmentRecord) -> Path:
+    sidecar_path = attachment_sidecar_path(artifact_path)
+    records: list[dict[str, object]] = []
+    if sidecar_path.exists():
+        records = json.loads(sidecar_path.read_text(encoding="utf-8"))
+    records.append(record.as_dict())
+    sidecar_path.write_text(json.dumps(records, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return sidecar_path

modelable/llm/providers.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from os import environ
+from typing import Any, Protocol, SupportsIndex, cast
+from urllib import error, request
+
+
+@dataclass(frozen=True)
+class LLMRequest:
+    system: str
+    user: str
+    temperature: float = 0.2
+    response_format: str = "text"
+    schema: dict[str, object] | None = None
+
+
+@dataclass(frozen=True)
+class LLMResponse:
+    content: str
+    provider: str
+    model: str
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+
+
+class LLMProvider(Protocol):
+    def complete(self, request: LLMRequest) -> LLMResponse:
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class OllamaProvider:
+    base_url: str
+    model: str
+    timeout: float = 120.0
+
+    def complete(self, request: LLMRequest) -> LLMResponse:
+        payload: dict[str, object] = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": request.system},
+                {"role": "user", "content": request.user},
+            ],
+            "stream": False,
+            "options": {"temperature": request.temperature},
+        }
+        if request.response_format == "json" or request.schema is not None:
+            payload["format"] = "json"
+
+        response = self._post_json("/api/chat", payload)
+        message = response.get("message") or {}
+        content = str(message.get("content") or "")
+        return LLMResponse(
+            content=content,
+            provider="ollama",
+            model=self.model,
+            prompt_tokens=_int_or_none(response.get("prompt_eval_count")),
+            completion_tokens=_int_or_none(response.get("eval_count")),
+        )
+
+    def _post_json(self, path: str, payload: dict[str, object]) -> dict[str, Any]:
+        body = json.dumps(payload).encode("utf-8")
+        req = request.Request(
+            self.base_url.rstrip("/") + path,
+            data=body,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with request.urlopen(req, timeout=self.timeout) as resp:
+                raw = resp.read().decode("utf-8")
+        except error.HTTPError as exc:  # pragma: no cover - thin transport wrapper
+            detail = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"Ollama request failed: {exc.code} {detail}") from exc
+        except error.URLError as exc:  # pragma: no cover - thin transport wrapper
+            raise RuntimeError(f"Ollama request failed: {exc.reason}") from exc
+
+        try:
+            return cast(dict[str, object], json.loads(raw))
+        except json.JSONDecodeError as exc:
+            raise RuntimeError(f"Ollama returned invalid JSON: {exc}") from exc
+
+
+def build_provider(provider: str | None, *, model: str | None, base_url: str | None) -> LLMProvider | None:
+    if provider is None:
+        return None
+    normalized = provider.strip().lower()
+    if normalized in {"local", "heuristic", "none"}:
+        return None
+    if normalized == "anthropic":
+        api_key = environ.get("ANTHROPIC_API_KEY")
+        if not api_key:
+            raise ValueError("anthropic provider requires ANTHROPIC_API_KEY")
+        if not model:
+            raise ValueError("anthropic provider requires a model")
+        return AnthropicProvider(api_key=api_key, model=model, base_url=base_url or "https://api.anthropic.com")
+    if normalized == "ollama":
+        if not model:
+            raise ValueError("ollama provider requires a model")
+        return OllamaProvider(base_url=base_url or "http://localhost:11434", model=model)
+    raise ValueError(f"Unsupported LLM provider: {provider}")
+
+
+@dataclass(frozen=True)
+class AnthropicProvider:
+    api_key: str
+    model: str
+    base_url: str = "https://api.anthropic.com"
+    timeout: float = 120.0
+
+    def complete(self, request: LLMRequest) -> LLMResponse:
+        payload: dict[str, object] = {
+            "model": self.model,
+            "max_tokens": 1024,
+            "system": request.system,
+            "messages": [
+                {"role": "user", "content": request.user},
+            ],
+        }
+        if request.response_format == "json" or request.schema is not None:
+            payload["max_tokens"] = 2048
+
+        response = self._post_json("/v1/messages", payload)
+        content = self._extract_content(response.get("content"))
+        usage = response.get("usage") if isinstance(response.get("usage"), dict) else None
+        return LLMResponse(
+            content=content,
+            provider="anthropic",
+            model=self.model,
+            prompt_tokens=_int_or_none(usage.get("input_tokens") if isinstance(usage, dict) else None),
+            completion_tokens=_int_or_none(usage.get("output_tokens") if isinstance(usage, dict) else None),
+        )
+
+    def _post_json(self, path: str, payload: dict[str, object]) -> dict[str, Any]:
+        body = json.dumps(payload).encode("utf-8")
+        req = request.Request(
+            self.base_url.rstrip("/") + path,
+            data=body,
+            headers={
+                "Content-Type": "application/json",
+                "x-api-key": self.api_key,
+                "anthropic-version": "2023-06-01",
+            },
+            method="POST",
+        )
+        try:
+            with request.urlopen(req, timeout=self.timeout) as resp:
+                raw = resp.read().decode("utf-8")
+        except error.HTTPError as exc:  # pragma: no cover - thin transport wrapper
+            detail = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"Anthropic request failed: {exc.code} {detail}") from exc
+        except error.URLError as exc:  # pragma: no cover - thin transport wrapper
+            raise RuntimeError(f"Anthropic request failed: {exc.reason}") from exc
+
+        try:
+            return cast(dict[str, object], json.loads(raw))
+        except json.JSONDecodeError as exc:
+            raise RuntimeError(f"Anthropic returned invalid JSON: {exc}") from exc
+
+    def _extract_content(self, content: object) -> str:
+        if isinstance(content, list):
+            parts: list[str] = []
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text":
+                    parts.append(str(block.get("text") or ""))
+            return "".join(parts)
+        if isinstance(content, str):
+            return content
+        return ""
+
+
+def _int_or_none(value: object) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(cast("int | str | bytes | bytearray | SupportsIndex", value))
+    except TypeError, ValueError:
+        return None

modelable/llm/qa.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from modelable.compiler.workspace import Workspace
+from modelable.llm.context import parse_model_ref
+
+
+def answer_question(workspace: Workspace, question: str) -> str:
+    lower = question.lower().strip()
+    if "owner" in lower or "who owns" in lower:
+        return _answer_owner_question(workspace, question)
+    if "lineage" in lower or "where did" in lower:
+        return _answer_lineage_question(workspace, question)
+    if "depend" in lower or "impact" in lower or "break" in lower:
+        return _answer_dependency_question(workspace, question)
+    if "field" in lower or "model" in lower:
+        return _answer_model_question(workspace, question)
+    return "I can answer ownership, lineage, dependency, and validation questions when you point me at a model or projection."
+
+
+def _answer_owner_question(workspace: Workspace, question: str) -> str:
+    ref = _extract_ref(question)
+    if ref:
+        domain = next((d for d in workspace.mdl.domains if d.name == ref.domain), None)
+        if domain is None:
+            return f"Unknown domain: {ref.domain}"
+        if ref.name in domain.models:
+            owner = domain.owner or "unspecified"
+            return f"{ref.domain}.{ref.name}@{ref.version} is owned by {owner}."
+        if ref.name in domain.projections:
+            owner = domain.owner or "unspecified"
+            return f"{ref.domain}.{ref.name}@{ref.version} is owned by {owner}."
+    return _domain_owners(workspace)
+
+
+def _answer_lineage_question(workspace: Workspace, question: str) -> str:
+    ref = _extract_ref(question)
+    if not ref:
+        return "Ask about a specific projection ref like `billing.BillingCustomer@1`."
+    domain = next((d for d in workspace.mdl.domains if d.name == ref.domain), None)
+    if domain is None:
+        return f"Unknown domain: {ref.domain}"
+    versions = domain.projections.get(ref.name)
+    if not versions:
+        return f"Unknown projection: {ref.domain}.{ref.name}"
+    version = next((item for item in versions if item.version == ref.version), None)
+    if version is None:
+        return f"Unknown projection version: {ref.domain}.{ref.name}@{ref.version}"
+    lines = [f"{ref.domain}.{ref.name}@{ref.version} lineage:"]
+    for field in version.fields:
+        lines.append(f"- {field.name}: {_field_lineage(field)}")
+    return "\n".join(lines)
+
+
+def _answer_dependency_question(workspace: Workspace, question: str) -> str:
+    ref = _extract_ref(question)
+    if not ref:
+        return "Ask about a specific model ref like `customer.Customer@1`."
+    dependents: list[str] = []
+    for domain in workspace.mdl.domains:
+        for projection_name, versions in domain.projections.items():
+            for version in versions:
+                if version.source.model == f"{ref.domain}.{ref.name}" or any(
+                    join.model == f"{ref.domain}.{ref.name}" for join in version.joins
+                ):
+                    dependents.append(f"{domain.name}.{projection_name}@{version.version}")
+    if not dependents:
+        return f"No projections currently depend on {ref.domain}.{ref.name}@{ref.version}."
+    return "Dependents:\n" + "\n".join(f"- {item}" for item in dependents)
+
+
+def _answer_model_question(workspace: Workspace, question: str) -> str:
+    ref = _extract_ref(question)
+    if ref:
+        return _model_summary(workspace, ref.domain, ref.name, ref.version)
+    return _workspace_summary(workspace)
+
+
+def _extract_ref(question: str):
+    tokens = question.replace("?", " ").replace(",", " ").split()
+    for token in tokens:
+        if "@" in token and "." in token:
+            try:
+                return parse_model_ref(token.strip("`"))
+            except Exception:
+                continue
+    return None
+
+
+def _model_summary(workspace: Workspace, domain_name: str, model_name: str, version: int) -> str:
+    domain = next((d for d in workspace.mdl.domains if d.name == domain_name), None)
+    if domain is None:
+        return f"Unknown domain: {domain_name}"
+    versions = domain.models.get(model_name)
+    if not versions:
+        return f"Unknown model: {domain_name}.{model_name}"
+    mv = next((item for item in versions if item.version == version), None)
+    if mv is None:
+        return f"Unknown model version: {domain_name}.{model_name}@{version}"
+    lines = [f"{domain_name}.{model_name}@{version} ({mv.model_kind.value}, {mv.change_kind.value})"]
+    if domain.owner:
+        lines.append(f"owner: {domain.owner}")
+    for field in mv.fields:
+        lines.append(f"- {field.name}")
+    return "\n".join(lines)
+
+
+def _domain_owners(workspace: Workspace) -> str:
+    lines = []
+    for domain in workspace.mdl.domains:
+        owner = domain.owner or "unspecified"
+        lines.append(f"{domain.name}: {owner}")
+    return "\n".join(lines) if lines else "No domains found."
+
+
+def _workspace_summary(workspace: Workspace) -> str:
+    lines = []
+    for domain in workspace.mdl.domains:
+        lines.append(f"{domain.name}: {len(domain.models)} models, {len(domain.projections)} projections")
+    return "\n".join(lines) if lines else "Workspace is empty."
+
+
+def _field_lineage(field) -> str:
+    mapping = field.mapping
+    if mapping.kind == "direct":
+        return f"direct {mapping.source_alias}.{mapping.source_field}"
+    return f"computed {mapping.expression}"

modelable/llm/recommendations.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from modelable.compiler.workspace import Workspace
+from modelable.llm.context import parse_model_ref
+
+
+def recommend_for_model(workspace: Workspace, ref: str | None = None, consumer: str | None = None) -> str:
+    if ref is None:
+        return "Provide a model ref like `customer.Customer@1` to get a targeted recommendation."
+    model_ref = parse_model_ref(ref)
+    domain = next((d for d in workspace.mdl.domains if d.name == model_ref.domain), None)
+    if domain is None:
+        return f"Unknown domain: {model_ref.domain}"
+    versions = domain.models.get(model_ref.name)
+    if not versions:
+        return f"Unknown model: {model_ref.domain}.{model_ref.name}"
+    version = next((item for item in versions if item.version == model_ref.version), None)
+    if version is None:
+        return f"Unknown model version: {ref}"
+
+    lines = [f"Recommendations for {ref}:"]
+    if consumer:
+        lines.append(f"- draft a projection for consumer `{consumer}` with explicit lineage")
+    if any(field.is_pii for field in version.fields):
+        lines.append("- preserve governance metadata or exclude PII from downstream projections")
+    if any(
+        field.classification and field.classification.value in {"confidential", "secret"} for field in version.fields
+    ):
+        lines.append("- keep classification on derived fields and avoid weakening it in projections")
+    if len(version.fields) > 10:
+        lines.append("- consider splitting the model into a smaller core entity plus value objects")
+    lines.append("- pin version references explicitly when generating imports or projections")
+    return "\n".join(lines)

modelable/llm/redaction.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import re
+
+_REDACTION_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"(?i)\b(password|passwd|secret|token|api[_-]?key)\s*=\s*([^\s,;]+)"), r"\1=[REDACTED]"),
+    (re.compile(r"(?i)\b(password|passwd|secret|token|api[_-]?key)\s*:\s*([^\s,;]+)"), r"\1: [REDACTED]"),
+    (
+        re.compile(r"(?i)(-----BEGIN [^-]+PRIVATE KEY-----).*?(-----END [^-]+PRIVATE KEY-----)", re.DOTALL),
+        "[REDACTED PRIVATE KEY]",
+    ),
+)
+
+
+def redact_sensitive_values(text: str) -> str:
+    redacted = text
+    for pattern, replacement in _REDACTION_PATTERNS:
+        redacted = pattern.sub(replacement, redacted)
+    return redacted