instructvault 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl

instructvault/cli.py

@@ -8,8 +8,10 @@ from rich import print as rprint
 from .bundle import write_bundle
 from .diff import unified_diff
 from .eval import run_dataset, run_inline_tests
-from .io import load_dataset_jsonl, load_prompt_spec
+from .io import load_dataset_jsonl, load_prompt_spec, load_prompt_dict
+from .policy import load_policy_module, run_spec_policy
 from .junit import write_junit_xml
+import yaml
 from .render import check_required_vars, render_messages
 from .scaffold import init_repo
 from .store import PromptStore
@@ -30,16 +32,25 @@ def init(repo: Path = typer.Option(Path("."), "--repo")):
 @app.command()
 def validate(path: Path = typer.Argument(...),
              repo: Path = typer.Option(Path("."), "--repo"),
-             json_out: bool = typer.Option(False, "--json")):
+             json_out: bool = typer.Option(False, "--json"),
+             policy: Optional[str] = typer.Option(None, "--policy")):
     base = path if path.is_absolute() else repo / path
     files = _gather_prompt_files(base)
     if not files:
         raise typer.BadParameter("No prompt files found")
     ok = True
     results = []
+    pol = load_policy_module(policy)
     for f in files:
         try:
             spec = load_prompt_spec(f.read_text(encoding="utf-8"), allow_no_tests=False)
+            errors = run_spec_policy(pol, load_prompt_dict(f.read_text(encoding="utf-8")))
+            if errors:
+                ok = False
+                results.append({"path": str(f), "ok": False, "error": "; ".join(errors)})
+                if not json_out:
+                    rprint(f"[red]FAIL[/red] {f}  {errors}")
+                continue
             try:
                 rel_path = f.relative_to(repo).as_posix()
             except ValueError:
@@ -66,15 +77,18 @@ def render(prompt_path: str = typer.Argument(...),
            ref: Optional[str] = typer.Option(None, "--ref"),
            repo: Path = typer.Option(Path("."), "--repo"),
            json_out: bool = typer.Option(False, "--json"),
-           allow_no_tests: bool = typer.Option(False, "--allow-no-tests")):
+           allow_no_tests: bool = typer.Option(False, "--allow-no-tests"),
+           safe: bool = typer.Option(False, "--safe"),
+           strict_vars: bool = typer.Option(False, "--strict-vars"),
+           redact: bool = typer.Option(False, "--redact")):
     store = PromptStore(repo_root=repo)
     spec = load_prompt_spec(store.read_text(prompt_path, ref=ref), allow_no_tests=allow_no_tests)
     try:
         vars_dict = json.loads(vars_json)
     except Exception:
         raise typer.BadParameter("Invalid JSON for --vars")
-    check_required_vars(spec, vars_dict)
-    msgs = render_messages(spec, vars_dict)
+    check_required_vars(spec, vars_dict, safe=safe, strict_vars=strict_vars, redact=redact)
+    msgs = render_messages(spec, vars_dict, safe=safe, strict_vars=strict_vars, redact=redact)
     if json_out:
         rprint(json.dumps([{"role": m.role, "content": m.content} for m in msgs]))
     else:
@@ -107,6 +121,32 @@ def resolve(ref: str = typer.Argument(...),
     else:
         rprint(sha)
 
+@app.command()
+def migrate(path: Path = typer.Argument(...),
+            repo: Path = typer.Option(Path("."), "--repo"),
+            apply: bool = typer.Option(False, "--apply")):
+    base = path if path.is_absolute() else repo / path
+    files = _gather_prompt_files(base)
+    if not files:
+        raise typer.BadParameter("No prompt files found")
+    needs = []
+    for f in files:
+        data = load_prompt_dict(f.read_text(encoding="utf-8"))
+        if "spec_version" not in data:
+            needs.append(f)
+    if not needs:
+        rprint("[green]No migration needed[/green]")
+        raise typer.Exit(code=0)
+    for f in needs:
+        if apply:
+            data = load_prompt_dict(f.read_text(encoding="utf-8"))
+            data["spec_version"] = "1.0"
+            f.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
+            rprint(f"[green]Updated:[/green] {f}")
+        else:
+            rprint(f"[yellow]Missing spec_version:[/yellow] {f}")
+    raise typer.Exit(code=0 if apply else 1)
+
 @app.command()
 def bundle(prompts: Path = typer.Option(Path("prompts"), "--prompts"),
            out: Path = typer.Option(Path("out/ivault.bundle.json"), "--out"),
@@ -123,17 +163,22 @@ def eval(prompt_path: str = typer.Argument(...),
          report: Optional[Path] = typer.Option(None, "--report"),
          junit: Optional[Path] = typer.Option(None, "--junit"),
          repo: Path = typer.Option(Path("."), "--repo"),
-         json_out: bool = typer.Option(False, "--json")):
+         json_out: bool = typer.Option(False, "--json"),
+         safe: bool = typer.Option(False, "--safe"),
+         strict_vars: bool = typer.Option(False, "--strict-vars"),
+         redact: bool = typer.Option(False, "--redact"),
+         policy: Optional[str] = typer.Option(None, "--policy")):
     store = PromptStore(repo_root=repo)
     spec = load_prompt_spec(store.read_text(prompt_path, ref=ref), allow_no_tests=False)
+    pol = load_policy_module(policy)
 
-    ok1, r1 = run_inline_tests(spec)
+    ok1, r1 = run_inline_tests(spec, safe=safe, strict_vars=strict_vars, redact=redact, policy=pol)
     results = list(r1)
     ok = ok1
 
     if dataset is not None:
         rows = load_dataset_jsonl(dataset.read_text(encoding="utf-8"))
-        ok2, r2 = run_dataset(spec, rows)
+        ok2, r2 = run_dataset(spec, rows, safe=safe, strict_vars=strict_vars, redact=redact, policy=pol)
         ok = ok and ok2
         results.extend(r2)
 

instructvault/eval.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
 from dataclasses import dataclass
 from typing import List, Optional, Tuple
+import json
+import re
 from .spec import AssertSpec, DatasetRow, PromptSpec
 from .render import check_required_vars, render_joined_text
+from .policy import run_render_policy
 
 @dataclass(frozen=True)
 class TestResult:
@@ -19,15 +22,34 @@ def _match_assert(assert_spec: AssertSpec, text: str) -> bool:
         ok = ok and all(s.lower() in t for s in assert_spec.contains_all)
     if assert_spec.not_contains:
         ok = ok and all(s.lower() not in t for s in assert_spec.not_contains)
+    if assert_spec.matches:
+        ok = ok and all(re.search(p, text) for p in assert_spec.matches)
+    if assert_spec.not_matches:
+        ok = ok and all(not re.search(p, text) for p in assert_spec.not_matches)
+    if assert_spec.json_schema:
+        try:
+            obj = json.loads(text)
+        except Exception:
+            return False
+        try:
+            import jsonschema  # type: ignore
+        except Exception as e:
+            raise ValueError("jsonschema is required for json_schema assertions") from e
+        jsonschema.validate(instance=obj, schema=assert_spec.json_schema)
     return ok
 
-def run_inline_tests(spec: PromptSpec) -> Tuple[bool, List[TestResult]]:
+def run_inline_tests(spec: PromptSpec, *, safe: bool = False, strict_vars: bool = False, redact: bool = False, policy: Optional[object] = None) -> Tuple[bool, List[TestResult]]:
     results: List[TestResult] = []
     all_ok = True
     for t in spec.tests:
         try:
-            check_required_vars(spec, t.vars)
-            out = render_joined_text(spec, t.vars)
+            check_required_vars(spec, t.vars, safe=safe, strict_vars=strict_vars, redact=redact)
+            out = render_joined_text(spec, t.vars, safe=safe, strict_vars=strict_vars, redact=redact)
+            errors = run_render_policy(policy, out, {"prompt": spec.name, "test": t.name, "kind": "inline"})
+            if errors:
+                results.append(TestResult(t.name, False, "; ".join(errors)))
+                all_ok = False
+                continue
             passed = _match_assert(t.assert_, out)
             results.append(TestResult(t.name, passed))
             all_ok = all_ok and passed
@@ -36,14 +58,19 @@ def run_inline_tests(spec: PromptSpec) -> Tuple[bool, List[TestResult]]:
             all_ok = False
     return all_ok, results
 
-def run_dataset(spec: PromptSpec, rows: List[DatasetRow]) -> Tuple[bool, List[TestResult]]:
+def run_dataset(spec: PromptSpec, rows: List[DatasetRow], *, safe: bool = False, strict_vars: bool = False, redact: bool = False, policy: Optional[object] = None) -> Tuple[bool, List[TestResult]]:
     results: List[TestResult] = []
     all_ok = True
     for i, row in enumerate(rows, start=1):
         name = f"dataset_row_{i}"
         try:
-            check_required_vars(spec, row.vars)
-            out = render_joined_text(spec, row.vars)
+            check_required_vars(spec, row.vars, safe=safe, strict_vars=strict_vars, redact=redact)
+            out = render_joined_text(spec, row.vars, safe=safe, strict_vars=strict_vars, redact=redact)
+            errors = run_render_policy(policy, out, {"prompt": spec.name, "test": name, "kind": "dataset"})
+            if errors:
+                results.append(TestResult(name, False, "; ".join(errors)))
+                all_ok = False
+                continue
             passed = _match_assert(row.assert_, out)
             results.append(TestResult(name, passed))
             all_ok = all_ok and passed

instructvault/io.py

@@ -15,6 +15,15 @@ def load_prompt_spec(yaml_text: str, *, allow_no_tests: bool = True) -> PromptSp
         data = yaml.safe_load(yaml_text) or {}
     return PromptSpec.model_validate(data, context={"allow_no_tests": allow_no_tests})
 
+def load_prompt_dict(text: str) -> Dict[str, Any]:
+    raw = text.strip()
+    if raw.startswith("{") or raw.startswith("["):
+        try:
+            return json.loads(raw) if raw else {}
+        except Exception:
+            return yaml.safe_load(text) or {}
+    return yaml.safe_load(text) or {}
+
 def load_dataset_jsonl(text: str) -> List[DatasetRow]:
     rows: List[DatasetRow] = []
     for i, line in enumerate(text.splitlines(), start=1):

instructvault/policy.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+import importlib.util
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+def load_policy_module(path: Optional[str]) -> Optional[object]:
+    if not path:
+        return None
+    p = Path(path)
+    spec = importlib.util.spec_from_file_location("ivault_policy", p)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Could not load policy module: {path}")
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+def run_spec_policy(mod: Optional[object], spec_dict: Dict[str, Any]) -> List[str]:
+    if mod is None:
+        return []
+    fn = getattr(mod, "check_spec", None)
+    if fn is None:
+        return []
+    res = fn(spec_dict)
+    return list(res) if res else []
+
+def run_render_policy(mod: Optional[object], text: str, context: Dict[str, Any]) -> List[str]:
+    if mod is None:
+        return []
+    fn = getattr(mod, "check_render", None)
+    if fn is None:
+        return []
+    res = fn(text, context)
+    return list(res) if res else []

instructvault/render.py

@@ -1,22 +1,57 @@
 from __future__ import annotations
 from typing import Any, Dict, List
+import re
 from jinja2 import Environment, StrictUndefined
 from .spec import PromptMessage, PromptSpec
 
 _env = Environment(undefined=StrictUndefined, autoescape=False)
 
-def check_required_vars(spec: PromptSpec, vars: Dict[str, Any]) -> None:
+_SECRET_PATTERNS = [
+    ("openai_key", re.compile(r"sk-[A-Za-z0-9]{20,}")),
+    ("aws_key", re.compile(r"AKIA[0-9A-Z]{16}")),
+    ("pypi_token", re.compile(r"pypi-[A-Za-z0-9]{20,}")),
+    ("generic_token", re.compile(r"(?:api|token|secret)[=_:\\s-]{1,}[A-Za-z0-9\\-]{16,}", re.IGNORECASE)),
+]
+
+def _scan_for_secrets(text: str) -> List[str]:
+    hits: List[str] = []
+    for name, pat in _SECRET_PATTERNS:
+        if pat.search(text):
+            hits.append(name)
+    return hits
+
+def check_required_vars(spec: PromptSpec, vars: Dict[str, Any], *, safe: bool = False, strict_vars: bool = False, redact: bool = False) -> None:
     missing = [k for k in spec.variables.required if k not in vars]
     if missing:
         raise ValueError(f"Missing required vars: {missing}")
+    if strict_vars:
+        allowed = set(spec.variables.required + spec.variables.optional)
+        extra = [k for k in vars.keys() if k not in allowed]
+        if extra:
+            raise ValueError(f"Unexpected vars: {extra}")
+    if safe and not redact:
+        for v in vars.values():
+            if isinstance(v, str):
+                hits = _scan_for_secrets(v)
+                if hits:
+                    raise ValueError(f"Potential secret detected in vars: {hits}")
 
-def render_messages(spec: PromptSpec, vars: Dict[str, Any]) -> List[PromptMessage]:
+def render_messages(spec: PromptSpec, vars: Dict[str, Any], *, safe: bool = False, strict_vars: bool = False, redact: bool = False) -> List[PromptMessage]:
     rendered: List[PromptMessage] = []
     for m in spec.messages:
         tmpl = _env.from_string(m.content)
-        rendered.append(PromptMessage(role=m.role, content=tmpl.render(**vars)))
+        content = tmpl.render(**vars)
+        if safe:
+            hits = _scan_for_secrets(content)
+            if hits:
+                if redact:
+                    for _, pat in _SECRET_PATTERNS:
+                        content = pat.sub("[REDACTED]", content)
+                else:
+                    raise ValueError(f"Potential secret detected in rendered output: {hits}")
+        rendered.append(PromptMessage(role=m.role, content=content))
     return rendered
 
-def render_joined_text(spec: PromptSpec, vars: Dict[str, Any]) -> str:
-    msgs = render_messages(spec, vars)
+def render_joined_text(spec: PromptSpec, vars: Dict[str, Any], *, safe: bool = False, strict_vars: bool = False, redact: bool = False) -> str:
+    msgs = render_messages(spec, vars, safe=safe, strict_vars=strict_vars, redact=redact)
     return "\n\n".join([f"{m.role}: {m.content}" for m in msgs])

instructvault/sdk.py

@@ -27,7 +27,7 @@ class InstructVault:
         if self.store is None:
             raise ValueError("No repo_root configured")
         return load_prompt_spec(self.store.read_text(prompt_path, ref=ref), allow_no_tests=True)
-    def render(self, prompt_path: str, vars: Dict[str, Any], ref: Optional[str] = None) -> List[PromptMessage]:
+    def render(self, prompt_path: str, vars: Dict[str, Any], ref: Optional[str] = None, *, safe: bool = False, strict_vars: bool = False, redact: bool = False) -> List[PromptMessage]:
         spec = self.load_prompt(prompt_path, ref=ref)
-        check_required_vars(spec, vars)
-        return render_messages(spec, vars)
+        check_required_vars(spec, vars, safe=safe, strict_vars=strict_vars, redact=redact)
+        return render_messages(spec, vars, safe=safe, strict_vars=strict_vars, redact=redact)

instructvault/spec.py

@@ -25,9 +25,12 @@ class AssertSpec(BaseModel):
     contains_any: Optional[List[str]] = None
     contains_all: Optional[List[str]] = None
     not_contains: Optional[List[str]] = None
+    matches: Optional[List[str]] = None
+    not_matches: Optional[List[str]] = None
+    json_schema: Optional[Dict[str, Any]] = None
     @model_validator(mode="after")
     def _require_one(self) -> "AssertSpec":
-        if not (self.contains_any or self.contains_all or self.not_contains):
+        if not (self.contains_any or self.contains_all or self.not_contains or self.matches or self.not_matches or self.json_schema):
             raise ValueError("assert must include at least one of contains_any, contains_all, not_contains")
         return self
 

{instructvault-0.2.8.dist-info → instructvault-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instructvault
-Version: 0.2.8
+Version: 0.3.0
 Summary: Git-first prompt registry + CI evals + lightweight runtime SDK (ivault).
 Project-URL: Homepage, https://github.com/05satyam/instruct_vault
 Project-URL: Repository, https://github.com/05satyam/instruct_vault
@@ -9,6 +9,7 @@ License: Apache-2.0
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: jinja2>=3.1
+Requires-Dist: jsonschema>=4.21
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=13.7
@@ -127,6 +128,10 @@ ivault validate prompts
 ivault render prompts/support_reply.prompt.yml --vars '{"ticket_text":"My app crashed.","customer_name":"Sam"}'
 ```
 
+Safety tip: add `--safe` to scan rendered output for common secret patterns.
+Use `--strict-vars` to forbid unknown vars and `--redact` to mask detected secrets.
+Use `--policy /path/to/policy.py` to enforce custom compliance rules.
+
 ### 4) Add dataset‑driven eval
 `datasets/support_cases.jsonl`
 ```jsonl
@@ -142,6 +147,11 @@ Note: Prompts must include at least one inline test. Datasets are optional.
 Migration tip: if you need to render a prompt that doesn’t yet include tests, use
 `ivault render --allow-no-tests` or add a minimal test first.
 
+Spec migration check:
+```bash
+ivault migrate prompts
+```
+
 ### 5) Version prompts with tags
 ```bash
 git add prompts datasets
@@ -179,6 +189,10 @@ vault = InstructVault(bundle_path="out/ivault.bundle.json")
 - `examples/notebooks/instructvault_openai_colab.ipynb`
   [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/05satyam/instruct_vault/blob/main/examples/notebooks/instructvault_openai_colab.ipynb)
 
+## Example Policies
+- `examples/policies/policy_example.py`
+- `examples/policies/policy_pack.py`
+
 ## How teams use this in production
 1) Prompt changes go through PRs
 2) CI runs `validate` + `eval`
@@ -208,11 +222,13 @@ Then send `x-ivault-api-key` in requests (or keep it behind your org gateway).
 If you don’t set the env var, no auth is required.
 
 ## Docs
+- `docs/spec.md`
 - `docs/vision.md`
 - `docs/governance.md`
 - `docs/ci.md`
 - `docs/playground.md`
 - `docs/cookbooks.md`
+- `docs/audit_logging.md`
 - `docs/dropin_guide.md`
 - `docs/release_checklist.md`
 - `docs/ci_templates/gitlab-ci.yml`

instructvault-0.3.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+instructvault/__init__.py,sha256=cg7j0qh6W84D-K0uSOLKKAP2JquW4NRXwZRDDLk5E18,59
+instructvault/bundle.py,sha256=6bfHNxJsE3zuZBLX5ZiMAhn1Dw6BnFHRa55fN6XIPRI,3008
+instructvault/cli.py,sha256=19oxI0RZcHSk3TyvaEaqWjrbB6DTRbtsb0M6b54dmTA,8566
+instructvault/diff.py,sha256=vz_vmKDXasNFoVKHCk2u_TsboHk1BdwvX0wCnJI1ATQ,252
+instructvault/eval.py,sha256=BgmJG7msEdCEW_UdpUpBdUlQ6F1yL8mtCnscwwCGCvc,3548
+instructvault/io.py,sha256=vlGaaEw5A8qnjsujxGjL9Xt2200-HEuYepm1gxQ1CKQ,1336
+instructvault/junit.py,sha256=sIEcIiGD3Xk6uCYjnE5p_07j8dPoS_RAc2eoy3BIBeQ,1133
+instructvault/policy.py,sha256=rQlPlVPWFudOCu6eXowFwGFZooUHplzi__cB_TrpLe4,1066
+instructvault/render.py,sha256=P4djgHlyuUDBapUD206z218JZxvbcXIO6mrHxH4ldKg,2525
+instructvault/scaffold.py,sha256=f5gwXE3dUPuJYTedZRqBs8w5SQEgt1dgDSuqW2dxrMg,1685
+instructvault/sdk.py,sha256=4M3d-KqyuWLcXkaccZjhI-BsBnD4cU_GynkJPJ3ged4,1901
+instructvault/spec.py,sha256=ybv-0rQ4Vqxrj277u7JAVttHQCGmix5zMmtZ4_SN7A8,2639
+instructvault/store.py,sha256=NhN49w7xrkeij0lQDr-CEdANYLpNVBXumv_cKqLmiYY,1056
+instructvault-0.3.0.dist-info/METADATA,sha256=po0yI-tsInHwHeJSokID1Vuvn9cKt67KgqbEMtsDSOg,7606
+instructvault-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+instructvault-0.3.0.dist-info/entry_points.txt,sha256=cdcMJQwBk9c95LwfN2W6x2xO43FwPjhfV3jHE7TTuHg,49
+instructvault-0.3.0.dist-info/licenses/LICENSE,sha256=VFbCvIsyizmkz4NrZPMdcPhyRK5uM0HhAjv3GBUbb7Y,135
+instructvault-0.3.0.dist-info/RECORD,,

instructvault-0.2.8.dist-info/RECORD

@@ -1,17 +0,0 @@
-instructvault/__init__.py,sha256=cg7j0qh6W84D-K0uSOLKKAP2JquW4NRXwZRDDLk5E18,59
-instructvault/bundle.py,sha256=6bfHNxJsE3zuZBLX5ZiMAhn1Dw6BnFHRa55fN6XIPRI,3008
-instructvault/cli.py,sha256=v5vP-sgVpXRs-YGvxH8VWIarFqUD1IsXdB9lseaFJDA,6310
-instructvault/diff.py,sha256=vz_vmKDXasNFoVKHCk2u_TsboHk1BdwvX0wCnJI1ATQ,252
-instructvault/eval.py,sha256=-yrFHCEUrONvzfKLP8s_RktFU74Ergp9tQJvzfrMR9s,1949
-instructvault/io.py,sha256=n1yQfiy93Duz-8tJ_HpbCEq8MUn2jlLpSmUY6XBg8G4,1037
-instructvault/junit.py,sha256=sIEcIiGD3Xk6uCYjnE5p_07j8dPoS_RAc2eoy3BIBeQ,1133
-instructvault/render.py,sha256=vcVnqIXGytskZEKbUofoKgIVflQSYhsmdpEtZs1X19A,919
-instructvault/scaffold.py,sha256=f5gwXE3dUPuJYTedZRqBs8w5SQEgt1dgDSuqW2dxrMg,1685
-instructvault/sdk.py,sha256=abqFrmc9Q5LUqC_ZrwM12DlpTZZkXqRuzN0T2x9lqqY,1727
-instructvault/spec.py,sha256=ZtVXosHy0f3hRB5CP9xbVzSdW8fDnf0-AR46ehG9-MA,2450
-instructvault/store.py,sha256=NhN49w7xrkeij0lQDr-CEdANYLpNVBXumv_cKqLmiYY,1056
-instructvault-0.2.8.dist-info/METADATA,sha256=R7vOJvngSwH-SOLTdiyDAPueNgmrUNnHHtN6NEpEsWI,7143
-instructvault-0.2.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-instructvault-0.2.8.dist-info/entry_points.txt,sha256=cdcMJQwBk9c95LwfN2W6x2xO43FwPjhfV3jHE7TTuHg,49
-instructvault-0.2.8.dist-info/licenses/LICENSE,sha256=VFbCvIsyizmkz4NrZPMdcPhyRK5uM0HhAjv3GBUbb7Y,135
-instructvault-0.2.8.dist-info/RECORD,,