instructvault 0.2.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ from .sdk import InstructVault
2
+ __all__ = ["InstructVault"]
@@ -0,0 +1,71 @@
1
+ from __future__ import annotations
2
+ import json
3
+ import subprocess
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+ from typing import Iterable, List, Optional
7
+
8
+ from .io import load_prompt_spec
9
+ from .spec import PromptSpec
10
+ from .store import PromptStore
11
+
12
+ @dataclass(frozen=True)
13
+ class BundlePrompt:
14
+ path: str
15
+ spec: PromptSpec
16
+
17
+ def _list_files_at_ref(repo_root: Path, ref: str, rel_dir: str) -> List[str]:
18
+ cmd = ["git", "-C", str(repo_root), "ls-tree", "-r", "--name-only", ref, rel_dir]
19
+ res = subprocess.run(cmd, capture_output=True, text=True)
20
+ if res.returncode != 0:
21
+ raise ValueError(res.stderr.strip() or f"Could not list files at ref {ref}")
22
+ return [line.strip() for line in res.stdout.splitlines() if line.strip()]
23
+
24
+ def _is_prompt_file(path: str) -> bool:
25
+ lower = path.lower()
26
+ return lower.endswith(".prompt.yml") or lower.endswith(".prompt.yaml") or lower.endswith(".prompt.json")
27
+
28
+ def collect_prompts(repo_root: Path, prompts_dir: Path, ref: Optional[str]) -> List[BundlePrompt]:
29
+ store = PromptStore(repo_root)
30
+ prompts: List[BundlePrompt] = []
31
+ if ref is None:
32
+ if not prompts_dir.exists():
33
+ raise FileNotFoundError(f"Prompts directory not found: {prompts_dir}")
34
+ try:
35
+ prompts_dir.relative_to(repo_root)
36
+ except Exception:
37
+ raise ValueError("prompts_dir must be within repo_root")
38
+ for p in sorted(prompts_dir.rglob("*.prompt.y*ml")):
39
+ rel_path = p.relative_to(repo_root).as_posix()
40
+ spec = load_prompt_spec(p.read_text(encoding="utf-8"), allow_no_tests=True)
41
+ prompts.append(BundlePrompt(rel_path, spec))
42
+ for p in sorted(prompts_dir.rglob("*.prompt.json")):
43
+ rel_path = p.relative_to(repo_root).as_posix()
44
+ spec = load_prompt_spec(p.read_text(encoding="utf-8"), allow_no_tests=True)
45
+ prompts.append(BundlePrompt(rel_path, spec))
46
+ if not prompts:
47
+ raise ValueError(f"No prompt files found in {prompts_dir}")
48
+ return prompts
49
+
50
+ rel_dir = prompts_dir.relative_to(repo_root).as_posix()
51
+ for rel_path in _list_files_at_ref(repo_root, ref, rel_dir):
52
+ if not _is_prompt_file(rel_path):
53
+ continue
54
+ spec = load_prompt_spec(store.read_text(rel_path, ref=ref), allow_no_tests=True)
55
+ prompts.append(BundlePrompt(rel_path, spec))
56
+ if not prompts:
57
+ raise ValueError(f"No prompt files found at ref {ref} in {rel_dir}")
58
+ return prompts
59
+
60
+ def write_bundle(out_path: Path, *, repo_root: Path, prompts_dir: Path, ref: Optional[str]) -> None:
61
+ prompts = collect_prompts(repo_root, prompts_dir, ref)
62
+ payload = {
63
+ "bundle_version": "1.0",
64
+ "ref": ref or "WORKTREE",
65
+ "prompts": [
66
+ {"path": p.path, "spec": p.spec.model_dump(by_alias=True)}
67
+ for p in prompts
68
+ ],
69
+ }
70
+ out_path.parent.mkdir(parents=True, exist_ok=True)
71
+ out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
instructvault/cli.py ADDED
@@ -0,0 +1,161 @@
1
+ from __future__ import annotations
2
+ import json
3
+ from pathlib import Path
4
+ from typing import Optional
5
+ import typer
6
+ from rich import print as rprint
7
+
8
+ from .bundle import write_bundle
9
+ from .diff import unified_diff
10
+ from .eval import run_dataset, run_inline_tests
11
+ from .io import load_dataset_jsonl, load_prompt_spec
12
+ from .junit import write_junit_xml
13
+ from .render import check_required_vars, render_messages
14
+ from .scaffold import init_repo
15
+ from .store import PromptStore
16
+
17
+ app = typer.Typer(help="InstructVault: git-first prompt registry + CI evals + runtime SDK")
18
+
19
+ def _gather_prompt_files(base: Path) -> list[Path]:
20
+ if base.is_file():
21
+ return [base]
22
+ files = sorted(base.rglob("*.prompt.y*ml")) + sorted(base.rglob("*.prompt.json"))
23
+ return files
24
+
25
+ @app.command()
26
+ def init(repo: Path = typer.Option(Path("."), "--repo")):
27
+ init_repo(repo)
28
+ rprint("[green]Initialized prompts/, datasets/, and .github/workflows/ivault.yml[/green]")
29
+
30
+ @app.command()
31
+ def validate(path: Path = typer.Argument(...),
32
+ repo: Path = typer.Option(Path("."), "--repo"),
33
+ json_out: bool = typer.Option(False, "--json")):
34
+ base = path if path.is_absolute() else repo / path
35
+ files = _gather_prompt_files(base)
36
+ if not files:
37
+ raise typer.BadParameter("No prompt files found")
38
+ ok = True
39
+ results = []
40
+ for f in files:
41
+ try:
42
+ spec = load_prompt_spec(f.read_text(encoding="utf-8"), allow_no_tests=False)
43
+ try:
44
+ rel_path = f.relative_to(repo).as_posix()
45
+ except ValueError:
46
+ rel_path = str(f)
47
+ results.append({"path": rel_path, "ok": True, "name": spec.name})
48
+ if not json_out:
49
+ rprint(f"[green]OK[/green] {rel_path} ({spec.name})")
50
+ except Exception as e:
51
+ ok = False
52
+ try:
53
+ rel_path = f.relative_to(repo).as_posix()
54
+ except ValueError:
55
+ rel_path = str(f)
56
+ results.append({"path": rel_path, "ok": False, "error": str(e)})
57
+ if not json_out:
58
+ rprint(f"[red]FAIL[/red] {rel_path} {e}")
59
+ if json_out:
60
+ rprint(json.dumps({"ok": ok, "results": results}))
61
+ raise typer.Exit(code=0 if ok else 1)
62
+
63
+ @app.command()
64
+ def render(prompt_path: str = typer.Argument(...),
65
+ vars_json: str = typer.Option("{}", "--vars"),
66
+ ref: Optional[str] = typer.Option(None, "--ref"),
67
+ repo: Path = typer.Option(Path("."), "--repo"),
68
+ json_out: bool = typer.Option(False, "--json"),
69
+ allow_no_tests: bool = typer.Option(False, "--allow-no-tests")):
70
+ store = PromptStore(repo_root=repo)
71
+ spec = load_prompt_spec(store.read_text(prompt_path, ref=ref), allow_no_tests=allow_no_tests)
72
+ try:
73
+ vars_dict = json.loads(vars_json)
74
+ except Exception:
75
+ raise typer.BadParameter("Invalid JSON for --vars")
76
+ check_required_vars(spec, vars_dict)
77
+ msgs = render_messages(spec, vars_dict)
78
+ if json_out:
79
+ rprint(json.dumps([{"role": m.role, "content": m.content} for m in msgs]))
80
+ else:
81
+ for m in msgs:
82
+ rprint(f"[bold]{m.role}[/bold]\n{m.content}\n")
83
+
84
+ @app.command()
85
+ def diff(prompt_path: str = typer.Argument(...),
86
+ ref1: str = typer.Option(..., "--ref1"),
87
+ ref2: str = typer.Option(..., "--ref2"),
88
+ repo: Path = typer.Option(Path("."), "--repo"),
89
+ json_out: bool = typer.Option(False, "--json")):
90
+ store = PromptStore(repo_root=repo)
91
+ a = store.read_text(prompt_path, ref=ref1)
92
+ b = store.read_text(prompt_path, ref=ref2)
93
+ d = unified_diff(a, b, f"{ref1}:{prompt_path}", f"{ref2}:{prompt_path}")
94
+ if json_out:
95
+ rprint(json.dumps({"same": not d.strip(), "diff": d}))
96
+ else:
97
+ rprint(d if d.strip() else "[yellow]No differences[/yellow]")
98
+
99
+ @app.command()
100
+ def resolve(ref: str = typer.Argument(...),
101
+ repo: Path = typer.Option(Path("."), "--repo"),
102
+ json_out: bool = typer.Option(False, "--json")):
103
+ store = PromptStore(repo_root=repo)
104
+ sha = store.resolve_ref(ref)
105
+ if json_out:
106
+ rprint(json.dumps({"ref": ref, "sha": sha}))
107
+ else:
108
+ rprint(sha)
109
+
110
+ @app.command()
111
+ def bundle(prompts: Path = typer.Option(Path("prompts"), "--prompts"),
112
+ out: Path = typer.Option(Path("out/ivault.bundle.json"), "--out"),
113
+ ref: Optional[str] = typer.Option(None, "--ref"),
114
+ repo: Path = typer.Option(Path("."), "--repo")):
115
+ prompts_dir = prompts if prompts.is_absolute() else repo / prompts
116
+ write_bundle(out, repo_root=repo, prompts_dir=prompts_dir, ref=ref)
117
+ rprint(f"[green]Wrote bundle[/green] {out}")
118
+
119
+ @app.command()
120
+ def eval(prompt_path: str = typer.Argument(...),
121
+ ref: Optional[str] = typer.Option(None, "--ref"),
122
+ dataset: Optional[Path] = typer.Option(None, "--dataset"),
123
+ report: Optional[Path] = typer.Option(None, "--report"),
124
+ junit: Optional[Path] = typer.Option(None, "--junit"),
125
+ repo: Path = typer.Option(Path("."), "--repo"),
126
+ json_out: bool = typer.Option(False, "--json")):
127
+ store = PromptStore(repo_root=repo)
128
+ spec = load_prompt_spec(store.read_text(prompt_path, ref=ref), allow_no_tests=False)
129
+
130
+ ok1, r1 = run_inline_tests(spec)
131
+ results = list(r1)
132
+ ok = ok1
133
+
134
+ if dataset is not None:
135
+ rows = load_dataset_jsonl(dataset.read_text(encoding="utf-8"))
136
+ ok2, r2 = run_dataset(spec, rows)
137
+ ok = ok and ok2
138
+ results.extend(r2)
139
+
140
+ payload = {
141
+ "prompt": spec.name,
142
+ "ref": ref or "WORKTREE",
143
+ "pass": ok,
144
+ "results": [{"test": r.name, "pass": r.passed, "error": r.error} for r in results],
145
+ }
146
+ if report:
147
+ report.parent.mkdir(parents=True, exist_ok=True)
148
+ report.write_text(json.dumps(payload, indent=2), encoding="utf-8")
149
+ if junit:
150
+ junit.parent.mkdir(parents=True, exist_ok=True)
151
+ write_junit_xml(suite_name=f"ivault:{spec.name}", results=results, out_path=str(junit))
152
+
153
+ if json_out:
154
+ rprint(json.dumps(payload))
155
+ else:
156
+ for r in results:
157
+ if r.passed:
158
+ rprint(f"[green]PASS[/green] {r.name}")
159
+ else:
160
+ rprint(f"[red]FAIL[/red] {r.name} {r.error or ''}")
161
+ raise typer.Exit(code=0 if ok else 1)
instructvault/diff.py ADDED
@@ -0,0 +1,4 @@
1
+ from __future__ import annotations
2
+ import difflib
3
+ def unified_diff(a: str, b: str, fromfile: str, tofile: str) -> str:
4
+ return "".join(difflib.unified_diff(a.splitlines(keepends=True), b.splitlines(keepends=True), fromfile=fromfile, tofile=tofile))
instructvault/eval.py ADDED
@@ -0,0 +1,53 @@
1
+ from __future__ import annotations
2
+ from dataclasses import dataclass
3
+ from typing import List, Optional, Tuple
4
+ from .spec import AssertSpec, DatasetRow, PromptSpec
5
+ from .render import check_required_vars, render_joined_text
6
+
7
+ @dataclass(frozen=True)
8
+ class TestResult:
9
+ name: str
10
+ passed: bool
11
+ error: Optional[str] = None
12
+
13
+ def _match_assert(assert_spec: AssertSpec, text: str) -> bool:
14
+ t = text.lower()
15
+ ok = True
16
+ if assert_spec.contains_any:
17
+ ok = ok and any(s.lower() in t for s in assert_spec.contains_any)
18
+ if assert_spec.contains_all:
19
+ ok = ok and all(s.lower() in t for s in assert_spec.contains_all)
20
+ if assert_spec.not_contains:
21
+ ok = ok and all(s.lower() not in t for s in assert_spec.not_contains)
22
+ return ok
23
+
24
+ def run_inline_tests(spec: PromptSpec) -> Tuple[bool, List[TestResult]]:
25
+ results: List[TestResult] = []
26
+ all_ok = True
27
+ for t in spec.tests:
28
+ try:
29
+ check_required_vars(spec, t.vars)
30
+ out = render_joined_text(spec, t.vars)
31
+ passed = _match_assert(t.assert_, out)
32
+ results.append(TestResult(t.name, passed))
33
+ all_ok = all_ok and passed
34
+ except Exception as e:
35
+ results.append(TestResult(t.name, False, str(e)))
36
+ all_ok = False
37
+ return all_ok, results
38
+
39
+ def run_dataset(spec: PromptSpec, rows: List[DatasetRow]) -> Tuple[bool, List[TestResult]]:
40
+ results: List[TestResult] = []
41
+ all_ok = True
42
+ for i, row in enumerate(rows, start=1):
43
+ name = f"dataset_row_{i}"
44
+ try:
45
+ check_required_vars(spec, row.vars)
46
+ out = render_joined_text(spec, row.vars)
47
+ passed = _match_assert(row.assert_, out)
48
+ results.append(TestResult(name, passed))
49
+ all_ok = all_ok and passed
50
+ except Exception as e:
51
+ results.append(TestResult(name, False, str(e)))
52
+ all_ok = False
53
+ return all_ok, results
instructvault/io.py ADDED
@@ -0,0 +1,29 @@
1
+ from __future__ import annotations
2
+ from typing import Any, Dict, List
3
+ import json
4
+ import yaml
5
+ from .spec import DatasetRow, PromptSpec
6
+
7
+ def load_prompt_spec(yaml_text: str, *, allow_no_tests: bool = True) -> PromptSpec:
8
+ text = yaml_text.strip()
9
+ if text.startswith("{") or text.startswith("["):
10
+ try:
11
+ data: Dict[str, Any] = json.loads(text) if text else {}
12
+ except Exception:
13
+ data = yaml.safe_load(yaml_text) or {}
14
+ else:
15
+ data = yaml.safe_load(yaml_text) or {}
16
+ return PromptSpec.model_validate(data, context={"allow_no_tests": allow_no_tests})
17
+
18
+ def load_dataset_jsonl(text: str) -> List[DatasetRow]:
19
+ rows: List[DatasetRow] = []
20
+ for i, line in enumerate(text.splitlines(), start=1):
21
+ line = line.strip()
22
+ if not line:
23
+ continue
24
+ try:
25
+ obj = json.loads(line)
26
+ except Exception as e:
27
+ raise ValueError(f"Invalid JSON on line {i}: {e}") from e
28
+ rows.append(DatasetRow.model_validate(obj))
29
+ return rows
instructvault/junit.py ADDED
@@ -0,0 +1,30 @@
1
+ from __future__ import annotations
2
+ import xml.etree.ElementTree as ET
3
+ from datetime import datetime, timezone
4
+ from typing import Iterable, Optional
5
+ from .eval import TestResult
6
+
7
+ def write_junit_xml(*, suite_name: str, results: Iterable[TestResult], out_path: str, timestamp: Optional[str] = None) -> None:
8
+ results = list(results)
9
+ tests = len(results)
10
+ failures = sum(1 for r in results if not r.passed)
11
+ ts = timestamp or datetime.now(timezone.utc).isoformat()
12
+
13
+ suite = ET.Element("testsuite", {
14
+ "name": suite_name,
15
+ "tests": str(tests),
16
+ "failures": str(failures),
17
+ "errors": "0",
18
+ "time": "0",
19
+ "timestamp": ts,
20
+ })
21
+
22
+ for r in results:
23
+ case = ET.SubElement(suite, "testcase", {"name": r.name, "classname": suite_name, "time": "0"})
24
+ if not r.passed:
25
+ f = ET.SubElement(case, "failure", {"message": r.error or "assertion failed", "type": "AssertionError"})
26
+ f.text = r.error or "assertion failed"
27
+
28
+ tree = ET.ElementTree(suite)
29
+ ET.indent(tree, space=" ")
30
+ tree.write(out_path, encoding="utf-8", xml_declaration=True)
@@ -0,0 +1,22 @@
1
+ from __future__ import annotations
2
+ from typing import Any, Dict, List
3
+ from jinja2 import Environment, StrictUndefined
4
+ from .spec import PromptMessage, PromptSpec
5
+
6
+ _env = Environment(undefined=StrictUndefined, autoescape=False)
7
+
8
+ def check_required_vars(spec: PromptSpec, vars: Dict[str, Any]) -> None:
9
+ missing = [k for k in spec.variables.required if k not in vars]
10
+ if missing:
11
+ raise ValueError(f"Missing required vars: {missing}")
12
+
13
+ def render_messages(spec: PromptSpec, vars: Dict[str, Any]) -> List[PromptMessage]:
14
+ rendered: List[PromptMessage] = []
15
+ for m in spec.messages:
16
+ tmpl = _env.from_string(m.content)
17
+ rendered.append(PromptMessage(role=m.role, content=tmpl.render(**vars)))
18
+ return rendered
19
+
20
+ def render_joined_text(spec: PromptSpec, vars: Dict[str, Any]) -> str:
21
+ msgs = render_messages(spec, vars)
22
+ return "\n\n".join([f"{m.role}: {m.content}" for m in msgs])
@@ -0,0 +1,62 @@
1
+ from __future__ import annotations
2
+ from pathlib import Path
3
+
4
+ DEFAULT_PROMPT = '''spec_version: "1.0"
5
+ name: hello_world
6
+ description: Minimal example prompt.
7
+ variables:
8
+ required: [name]
9
+ messages:
10
+ - role: system
11
+ content: "You are a helpful assistant."
12
+ - role: user
13
+ content: "Say hello to {{ name }}."
14
+ tests:
15
+ - name: includes_name
16
+ vars: { name: "Ava" }
17
+ assert: { contains_any: ["Ava"] }
18
+ '''
19
+
20
+ DEFAULT_WORKFLOW = '''name: ivault (prompts)
21
+ on:
22
+ pull_request:
23
+ paths:
24
+ - "prompts/**"
25
+ - "datasets/**"
26
+ push:
27
+ branches: [ main ]
28
+ paths:
29
+ - "prompts/**"
30
+ - "datasets/**"
31
+
32
+ jobs:
33
+ prompts:
34
+ runs-on: ubuntu-latest
35
+ steps:
36
+ - uses: actions/checkout@v4
37
+ - uses: actions/setup-python@v5
38
+ with:
39
+ python-version: "3.11"
40
+ - run: pip install instructvault
41
+ - run: ivault validate prompts
42
+ - run: ivault eval prompts/hello_world.prompt.yml --report out/report.json --junit out/junit.xml
43
+ - uses: actions/upload-artifact@v4
44
+ if: always()
45
+ with:
46
+ name: ivault-reports
47
+ path: out/
48
+ '''
49
+
50
+ def init_repo(repo_root: Path) -> None:
51
+ prompts = repo_root / "prompts"
52
+ datasets = repo_root / "datasets"
53
+ workflow = repo_root / ".github" / "workflows" / "ivault.yml"
54
+ prompts.mkdir(parents=True, exist_ok=True)
55
+ datasets.mkdir(parents=True, exist_ok=True)
56
+ workflow.parent.mkdir(parents=True, exist_ok=True)
57
+
58
+ sample_prompt = prompts / "hello_world.prompt.yml"
59
+ if not sample_prompt.exists():
60
+ sample_prompt.write_text(DEFAULT_PROMPT, encoding="utf-8")
61
+ if not workflow.exists():
62
+ workflow.write_text(DEFAULT_WORKFLOW, encoding="utf-8")
instructvault/sdk.py ADDED
@@ -0,0 +1,33 @@
1
+ from __future__ import annotations
2
+ from pathlib import Path
3
+ from typing import Any, Dict, List, Optional, Union
4
+ import json
5
+ from .io import load_prompt_spec
6
+ from .render import check_required_vars, render_messages
7
+ from .spec import PromptMessage, PromptSpec
8
+ from .store import PromptStore
9
+
10
+ class InstructVault:
11
+ def __init__(self, repo_root: Optional[Union[str, Path]] = None, bundle_path: Optional[Union[str, Path]] = None):
12
+ if repo_root is None and bundle_path is None:
13
+ raise ValueError("Provide repo_root or bundle_path")
14
+ self.store = PromptStore(Path(repo_root)) if repo_root is not None else None
15
+ self.bundle = None
16
+ if bundle_path is not None:
17
+ data = json.loads(Path(bundle_path).read_text(encoding="utf-8"))
18
+ self.bundle = {p["path"]: PromptSpec.model_validate(p["spec"]) for p in data.get("prompts", [])}
19
+
20
+ def load_prompt(self, prompt_path: str, ref: Optional[str] = None) -> PromptSpec:
21
+ if self.bundle is not None:
22
+ if ref is not None:
23
+ raise ValueError("ref is not supported when using bundle_path")
24
+ if prompt_path not in self.bundle:
25
+ raise FileNotFoundError(f"Prompt not found in bundle: {prompt_path}")
26
+ return self.bundle[prompt_path]
27
+ if self.store is None:
28
+ raise ValueError("No repo_root configured")
29
+ return load_prompt_spec(self.store.read_text(prompt_path, ref=ref), allow_no_tests=True)
30
+ def render(self, prompt_path: str, vars: Dict[str, Any], ref: Optional[str] = None) -> List[PromptMessage]:
31
+ spec = self.load_prompt(prompt_path, ref=ref)
32
+ check_required_vars(spec, vars)
33
+ return render_messages(spec, vars)
instructvault/spec.py ADDED
@@ -0,0 +1,64 @@
1
+ from __future__ import annotations
2
+ from typing import Any, Dict, List, Literal, Optional
3
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
4
+
5
+ Role = Literal["system", "user", "assistant", "tool"]
6
+
7
+ class PromptMessage(BaseModel):
8
+ model_config = ConfigDict(extra="forbid")
9
+ role: Role
10
+ content: str
11
+
12
+ class VariableSpec(BaseModel):
13
+ model_config = ConfigDict(extra="forbid")
14
+ required: List[str] = Field(default_factory=list)
15
+ optional: List[str] = Field(default_factory=list)
16
+
17
+ class ModelDefaults(BaseModel):
18
+ model_config = ConfigDict(extra="allow")
19
+ temperature: Optional[float] = None
20
+ top_p: Optional[float] = None
21
+ max_tokens: Optional[int] = None
22
+
23
+ class AssertSpec(BaseModel):
24
+ model_config = ConfigDict(extra="forbid")
25
+ contains_any: Optional[List[str]] = None
26
+ contains_all: Optional[List[str]] = None
27
+ not_contains: Optional[List[str]] = None
28
+ @model_validator(mode="after")
29
+ def _require_one(self) -> "AssertSpec":
30
+ if not (self.contains_any or self.contains_all or self.not_contains):
31
+ raise ValueError("assert must include at least one of contains_any, contains_all, not_contains")
32
+ return self
33
+
34
+ class PromptTest(BaseModel):
35
+ model_config = ConfigDict(extra="forbid")
36
+ name: str
37
+ vars: Dict[str, Any] = Field(default_factory=dict)
38
+ assert_: AssertSpec = Field(alias="assert")
39
+
40
+ class PromptSpec(BaseModel):
41
+ model_config = ConfigDict(extra="forbid", populate_by_name=True)
42
+ spec_version: str = Field(default="1.0", alias="spec_version")
43
+ name: str
44
+ description: Optional[str] = None
45
+ model_defaults: ModelDefaults = Field(default_factory=ModelDefaults, alias="modelParameters")
46
+ variables: VariableSpec = Field(default_factory=VariableSpec)
47
+ messages: List[PromptMessage]
48
+ tests: List[PromptTest] = Field(default_factory=list)
49
+
50
+ @model_validator(mode="after")
51
+ def _require_tests(self) -> "PromptSpec":
52
+ allow_no_tests = False
53
+ try:
54
+ allow_no_tests = bool(self.__pydantic_context__.get("allow_no_tests"))
55
+ except Exception:
56
+ allow_no_tests = False
57
+ if not self.tests and not allow_no_tests:
58
+ raise ValueError("prompt must include at least one test")
59
+ return self
60
+
61
+ class DatasetRow(BaseModel):
62
+ model_config = ConfigDict(extra="forbid")
63
+ vars: Dict[str, Any] = Field(default_factory=dict)
64
+ assert_: AssertSpec = Field(alias="assert")
instructvault/store.py ADDED
@@ -0,0 +1,25 @@
1
+ from __future__ import annotations
2
+ import subprocess
3
+ from pathlib import Path
4
+ from typing import Optional
5
+
6
+ class PromptStore:
7
+ def __init__(self, repo_root: Path):
8
+ self.repo_root = repo_root.resolve()
9
+
10
+ def read_text(self, rel_path: str, ref: Optional[str] = None) -> str:
11
+ rel_path = rel_path.lstrip("/")
12
+ if ref is None:
13
+ return (self.repo_root / rel_path).read_text(encoding="utf-8")
14
+ cmd = ["git", "-C", str(self.repo_root), "show", f"{ref}:{rel_path}"]
15
+ res = subprocess.run(cmd, capture_output=True, text=True)
16
+ if res.returncode != 0:
17
+ raise FileNotFoundError(res.stderr.strip() or f"Could not read {rel_path} at ref {ref}")
18
+ return res.stdout
19
+
20
+ def resolve_ref(self, ref: str) -> str:
21
+ cmd = ["git", "-C", str(self.repo_root), "rev-parse", ref]
22
+ res = subprocess.run(cmd, capture_output=True, text=True)
23
+ if res.returncode != 0:
24
+ raise ValueError(res.stderr.strip() or f"Could not resolve ref {ref}")
25
+ return res.stdout.strip()
@@ -0,0 +1,212 @@
1
+ Metadata-Version: 2.4
2
+ Name: instructvault
3
+ Version: 0.2.4
4
+ Summary: Git-first prompt registry + CI evals + lightweight runtime SDK (ivault).
5
+ Project-URL: Homepage, https://github.com/05satyam/instruct_vault
6
+ Project-URL: Repository, https://github.com/05satyam/instruct_vault
7
+ Author: InstructVault OSS Maintainers
8
+ License: Apache-2.0
9
+ License-File: LICENSE
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: jinja2>=3.1
12
+ Requires-Dist: pydantic>=2.7
13
+ Requires-Dist: pyyaml>=6.0
14
+ Requires-Dist: rich>=13.7
15
+ Requires-Dist: typer>=0.12
16
+ Provides-Extra: dev
17
+ Requires-Dist: fastapi>=0.110; extra == 'dev'
18
+ Requires-Dist: httpx>=0.27; extra == 'dev'
19
+ Requires-Dist: mypy>=1.10; extra == 'dev'
20
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
21
+ Requires-Dist: pytest>=8.0; extra == 'dev'
22
+ Requires-Dist: ruff>=0.6; extra == 'dev'
23
+ Requires-Dist: types-pyyaml; extra == 'dev'
24
+ Description-Content-Type: text/markdown
25
+
26
+ # InstructVault (`ivault`)
27
+ **Git‑first prompt hub for teams and individual developers.**
28
+
29
+ InstructVault makes prompts **first‑class, governed, testable, versioned artifacts** — just like code — while keeping runtime **fast and local**.
30
+
31
+ ## What this does (at a glance)
32
+ - **Prompts live in Git** as YAML/JSON files
33
+ - **CI validates + evaluates** prompts on every change
34
+ - **Releases are tags/SHAs**, reproducible by design
35
+ - **Runtime stays lightweight** (local read or bundle artifact)
36
+
37
+ ## System flow (Mermaid)
38
+ ```mermaid
39
+ flowchart LR
40
+ A[Prompt files<br/>YAML/JSON] --> B[PR Review]
41
+ B --> C[CI: validate + eval]
42
+ C --> D{Release?}
43
+ D -- tag/SHA --> E[Bundle artifact]
44
+ D -- tag/SHA --> F[Deploy app]
45
+ E --> F
46
+ F --> G[Runtime render<br/>local or bundle]
47
+ ```
48
+
49
+ ## Why this exists
50
+ Enterprises already have Git + PR reviews + CI/CD. Prompts usually don’t.
51
+ InstructVault brings **prompt‑as‑code** without requiring a server, database, or platform.
52
+
53
+ ## Vision
54
+ Short version: Git‑first prompts with CI governance and zero‑latency runtime.
55
+ Full vision: `docs/vision.md`
56
+
57
+ ## Features
58
+ - ✅ Git‑native versioning (tags/SHAs = releases)
59
+ - ✅ CLI‑first (`init`, `validate`, `render`, `eval`, `diff`, `resolve`, `bundle`)
60
+ - ✅ LLM‑framework agnostic (returns standard `{role, content}` messages)
61
+ - ✅ CI‑friendly reports (JSON + optional JUnit XML)
62
+ - ✅ No runtime latency tax (local read or bundle)
63
+ - ✅ Optional playground (separate package)
64
+
65
+ ## Install
66
+ ### Users
67
+ ```bash
68
+ pip install instructvault
69
+ ```
70
+
71
+ ### Contributors
72
+ ```bash
73
+ git clone <your-repo>
74
+ cd instructvault
75
+ python -m venv .venv
76
+ source .venv/bin/activate
77
+ pip install -e ".[dev]"
78
+ pytest
79
+ ```
80
+
81
+ ## Quickstart (end‑to‑end)
82
+
83
+ ### 1) Initialize a repo
84
+ ```bash
85
+ ivault init
86
+ ```
87
+
88
+ ### 2) Create a prompt
89
+ `prompts/support_reply.prompt.yml` (YAML or JSON)
90
+ ```yaml
91
+ spec_version: "1.0"
92
+ name: support_reply
93
+ description: Respond to a support ticket with empathy and clear steps.
94
+ model_defaults:
95
+ temperature: 0.2
96
+
97
+ variables:
98
+ required: [ticket_text]
99
+ optional: [customer_name]
100
+
101
+ messages:
102
+ - role: system
103
+ content: |
104
+ You are a support engineer. Be concise, empathetic, and action-oriented.
105
+ - role: user
106
+ content: |
107
+ Customer: {{ customer_name | default("there") }}
108
+ Ticket:
109
+ {{ ticket_text }}
110
+
111
+ tests:
112
+ - name: must_contain_customer_and_ticket
113
+ vars:
114
+ ticket_text: "My order arrived damaged."
115
+ customer_name: "Alex"
116
+ assert:
117
+ contains_all: ["Customer:", "Ticket:"]
118
+ ```
119
+
120
+ ### 3) Validate + render locally
121
+ ```bash
122
+ ivault validate prompts
123
+ ivault render prompts/support_reply.prompt.yml --vars '{"ticket_text":"My app crashed.","customer_name":"Sam"}'
124
+ ```
125
+
126
+ ### 4) Add dataset‑driven eval
127
+ `datasets/support_cases.jsonl`
128
+ ```jsonl
129
+ {"vars":{"ticket_text":"Order arrived damaged","customer_name":"Alex"},"assert":{"contains_any":["Ticket:"]}}
130
+ {"vars":{"ticket_text":"Need refund"},"assert":{"contains_all":["Ticket:"]}}
131
+ ```
132
+
133
+ ```bash
134
+ ivault eval prompts/support_reply.prompt.yml --dataset datasets/support_cases.jsonl --report out/report.json --junit out/junit.xml
135
+ ```
136
+
137
+ Note: Prompts must include at least one inline test. Datasets are optional.
138
+ Migration tip: if you need to render a prompt that doesn’t yet include tests, use
139
+ `ivault render --allow-no-tests` or add a minimal test first.
140
+
141
+ ### 5) Version prompts with tags
142
+ ```bash
143
+ git add prompts datasets
144
+ git commit -m "Add support prompts + eval dataset"
145
+ git tag prompts/v1.0.0
146
+ ```
147
+
148
+ ### 6) Load by Git ref at runtime
149
+ ```python
150
+ from instructvault import InstructVault
151
+
152
+ vault = InstructVault(repo_root=".")
153
+ msgs = vault.render(
154
+ "prompts/support_reply.prompt.yml",
155
+ vars={"ticket_text":"My order is delayed", "customer_name":"Ava"},
156
+ ref="prompts/v1.0.0",
157
+ )
158
+ ```
159
+
160
+ ### 7) Bundle prompts at build time (optional)
161
+ ```bash
162
+ ivault bundle --prompts prompts --out out/ivault.bundle.json --ref prompts/v1.0.0
163
+ ```
164
+
165
+ ```python
166
+ from instructvault import InstructVault
167
+ vault = InstructVault(bundle_path="out/ivault.bundle.json")
168
+ ```
169
+
170
+ ## How teams use this in production
171
+ 1) Prompt changes go through PRs
172
+ 2) CI runs `validate` + `eval`
173
+ 3) Tags or bundles become the deployable artifact
174
+ 4) Apps load by tag or bundle (no runtime network calls)
175
+
176
+ ## Datasets (why JSONL)
177
+ Datasets are **deterministic eval inputs** checked into Git. This makes CI reproducible and audit‑friendly.
178
+ For cloud datasets, use a CI pre‑step (e.g., download from S3) and then run `ivault eval` on the local file.
179
+
180
+ ## Playground (optional)
181
+ A minimal playground exists under `playground/` for local or org‑hosted use.
182
+ It lists prompts, renders with variables, and runs evals — without touching production prompts directly.
183
+ For local dev, run from the repo root:
184
+ ```bash
185
+ export IVAULT_REPO_ROOT=/path/to/your/repo
186
+ PYTHONPATH=. uvicorn ivault_playground.app:app --reload
187
+ ```
188
+
189
+ ![Playground screenshot](docs/assets/playground.png)
190
+
191
+ Optional auth:
192
+ ```bash
193
+ export IVAULT_PLAYGROUND_API_KEY=your-secret
194
+ ```
195
+ Then send `x-ivault-api-key` in requests (or keep it behind your org gateway).
196
+ If you don’t set the env var, no auth is required.
197
+
198
+ ## Docs
199
+ - `docs/vision.md`
200
+ - `docs/governance.md`
201
+ - `docs/ci.md`
202
+ - `docs/playground.md`
203
+ - `docs/cookbooks.md`
204
+ - `docs/dropin_guide.md`
205
+ - `docs/release_checklist.md`
206
+ - `docs/ci_templates/gitlab-ci.yml`
207
+ - `docs/ci_templates/Jenkinsfile`
208
+ - `CHANGELOG.md`
209
+ - `CODE_OF_CONDUCT.md`
210
+
211
+ ## License
212
+ Apache‑2.0
@@ -0,0 +1,17 @@
1
+ instructvault/__init__.py,sha256=cg7j0qh6W84D-K0uSOLKKAP2JquW4NRXwZRDDLk5E18,59
2
+ instructvault/bundle.py,sha256=6bfHNxJsE3zuZBLX5ZiMAhn1Dw6BnFHRa55fN6XIPRI,3008
3
+ instructvault/cli.py,sha256=v5vP-sgVpXRs-YGvxH8VWIarFqUD1IsXdB9lseaFJDA,6310
4
+ instructvault/diff.py,sha256=vz_vmKDXasNFoVKHCk2u_TsboHk1BdwvX0wCnJI1ATQ,252
5
+ instructvault/eval.py,sha256=-yrFHCEUrONvzfKLP8s_RktFU74Ergp9tQJvzfrMR9s,1949
6
+ instructvault/io.py,sha256=n1yQfiy93Duz-8tJ_HpbCEq8MUn2jlLpSmUY6XBg8G4,1037
7
+ instructvault/junit.py,sha256=sIEcIiGD3Xk6uCYjnE5p_07j8dPoS_RAc2eoy3BIBeQ,1133
8
+ instructvault/render.py,sha256=vcVnqIXGytskZEKbUofoKgIVflQSYhsmdpEtZs1X19A,919
9
+ instructvault/scaffold.py,sha256=f5gwXE3dUPuJYTedZRqBs8w5SQEgt1dgDSuqW2dxrMg,1685
10
+ instructvault/sdk.py,sha256=abqFrmc9Q5LUqC_ZrwM12DlpTZZkXqRuzN0T2x9lqqY,1727
11
+ instructvault/spec.py,sha256=ZtVXosHy0f3hRB5CP9xbVzSdW8fDnf0-AR46ehG9-MA,2450
12
+ instructvault/store.py,sha256=NhN49w7xrkeij0lQDr-CEdANYLpNVBXumv_cKqLmiYY,1056
13
+ instructvault-0.2.4.dist-info/METADATA,sha256=P5ZMGsVHnRNki6huaShXUduiTNzQVeAEByRsDlGGJAs,6251
14
+ instructvault-0.2.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
15
+ instructvault-0.2.4.dist-info/entry_points.txt,sha256=cdcMJQwBk9c95LwfN2W6x2xO43FwPjhfV3jHE7TTuHg,49
16
+ instructvault-0.2.4.dist-info/licenses/LICENSE,sha256=VFbCvIsyizmkz4NrZPMdcPhyRK5uM0HhAjv3GBUbb7Y,135
17
+ instructvault-0.2.4.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.28.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ ivault = instructvault.cli:app
@@ -0,0 +1,5 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION