rubricloop 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rubricloop-0.1.2 → rubricloop-0.1.3}/PKG-INFO +2 -1
- {rubricloop-0.1.2 → rubricloop-0.1.3}/README.md +1 -0
- rubricloop-0.1.3/examples/verify_reply_registry/README.md +47 -0
- rubricloop-0.1.3/examples/verify_reply_registry/build_package.py +122 -0
- rubricloop-0.1.3/examples/verify_reply_registry/run-input.json +6 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/pyproject.toml +1 -1
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/__init__.py +1 -1
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/cli.py +18 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_registry_cli.py +48 -1
- {rubricloop-0.1.2 → rubricloop-0.1.3}/.gitignore +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/LICENSE +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/RLPACK.md +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/VETTING.md +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/examples/sql_agent.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/examples/verify_reply.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/checks/__init__.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/checks/common.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/checks/sql.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/checks/structured.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/examples/__init__.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/examples/verify_reply.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/loop.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/__init__.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/base.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/catalog.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/commerce.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/finance.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/logistics.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/official.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/refund.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/security.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/sql.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/support.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/packs/vetting.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/py.typed +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/registry.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/rlpack.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/rubric.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/sandboxes/__init__.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/sandboxes/sqlite.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/src/rubricloop/types.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_common_checks.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_invariant.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_loop.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_refund_pack.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_rlpack.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_sql_pack.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_vetting_scenarios.py +0 -0
- {rubricloop-0.1.2 → rubricloop-0.1.3}/tests/test_wave_one_packs.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: rubricloop
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Bounded, auditable verification loops for AI agent output.
|
|
5
5
|
Project-URL: Homepage, https://rubricloop.com
|
|
6
6
|
Project-URL: Documentation, https://rubricloop.com/docs
|
|
@@ -180,6 +180,7 @@ hosted registry:
|
|
|
180
180
|
|
|
181
181
|
```bash
|
|
182
182
|
rubricloop --server https://api.rubricloop.dev login --api-key "$RUBRICLOOP_API_KEY"
|
|
183
|
+
rubricloop --server https://api.rubricloop.dev create acme/check --visibility private
|
|
183
184
|
rubricloop --server https://api.rubricloop.dev push dist/acme-check.rlpack --tag latest
|
|
184
185
|
rubricloop --server https://api.rubricloop.dev vet acme/check@latest
|
|
185
186
|
rubricloop --server https://api.rubricloop.dev pull acme/check@latest
|
|
@@ -151,6 +151,7 @@ hosted registry:
|
|
|
151
151
|
|
|
152
152
|
```bash
|
|
153
153
|
rubricloop --server https://api.rubricloop.dev login --api-key "$RUBRICLOOP_API_KEY"
|
|
154
|
+
rubricloop --server https://api.rubricloop.dev create acme/check --visibility private
|
|
154
155
|
rubricloop --server https://api.rubricloop.dev push dist/acme-check.rlpack --tag latest
|
|
155
156
|
rubricloop --server https://api.rubricloop.dev vet acme/check@latest
|
|
156
157
|
rubricloop --server https://api.rubricloop.dev pull acme/check@latest
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Reply Quick Start registry lifecycle
|
|
2
|
+
|
|
3
|
+
This package uses the same three deterministic rules as
|
|
4
|
+
`sdk/examples/verify_reply.py`: 40 to 70 words, the word `timeline`, and no
|
|
5
|
+
`guaranteed refund` promise.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install rubricloop openai
|
|
9
|
+
export OPENAI_API_KEY="sk-..."
|
|
10
|
+
python sdk/examples/verify_reply.py
|
|
11
|
+
|
|
12
|
+
python sdk/examples/verify_reply_registry/build_package.py \
|
|
13
|
+
--namespace YOUR_NAMESPACE
|
|
14
|
+
|
|
15
|
+
rubricloop --server https://api.rubricloop.dev login
|
|
16
|
+
rubricloop --server https://api.rubricloop.dev create \
|
|
17
|
+
YOUR_NAMESPACE/support-reply \
|
|
18
|
+
--visibility private \
|
|
19
|
+
--description "Checks support replies before they are sent."
|
|
20
|
+
rubricloop validate dist/support-reply.rlpack
|
|
21
|
+
rubricloop test dist/support-reply.rlpack
|
|
22
|
+
rubricloop --server https://api.rubricloop.dev push \
|
|
23
|
+
dist/support-reply.rlpack \
|
|
24
|
+
--tag latest
|
|
25
|
+
rubricloop --server https://api.rubricloop.dev vet \
|
|
26
|
+
YOUR_NAMESPACE/support-reply@latest
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Pull the immutable bundle digest returned by `push`, not the mutable tag:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
rubricloop --server https://api.rubricloop.dev pull \
|
|
33
|
+
YOUR_NAMESPACE/support-reply@sha256:BUNDLE_DIGEST \
|
|
34
|
+
--output dist/pulled-support-reply.rlpack
|
|
35
|
+
cmp dist/support-reply.rlpack dist/pulled-support-reply.rlpack
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Once vetting passes and hosted execution is approved for that exact digest:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
rubricloop --server https://api.rubricloop.dev run \
|
|
42
|
+
YOUR_NAMESPACE/support-reply@sha256:BUNDLE_DIGEST \
|
|
43
|
+
--input sdk/examples/verify_reply_registry/run-input.json
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
No credential belongs in this folder. `login` reads the key without echoing it
|
|
47
|
+
and stores it in the operating system keychain.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Build the registry package used by the OpenAI reply Quick Start."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from rubricloop import build_rlpack, create_manifest, read_rlpack
|
|
10
|
+
|
|
11
|
+
VALID_REPLY = (
|
|
12
|
+
"We apologize for the delay with your order. Our team is currently investigating "
|
|
13
|
+
"the issue and working to resolve it as quickly as possible. You can expect an "
|
|
14
|
+
"update on the timeline for your order shortly. Thank you for your patience."
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def manifest(namespace: str, version: str) -> dict:
|
|
19
|
+
golden = [
|
|
20
|
+
{
|
|
21
|
+
"id": f"valid-reply-{index:02d}",
|
|
22
|
+
"input": {"reply": VALID_REPLY},
|
|
23
|
+
"context": {},
|
|
24
|
+
"expected": {"decision": "allow"},
|
|
25
|
+
}
|
|
26
|
+
for index in range(1, 21)
|
|
27
|
+
]
|
|
28
|
+
return create_manifest(
|
|
29
|
+
namespace=namespace,
|
|
30
|
+
name="support-reply",
|
|
31
|
+
version=version,
|
|
32
|
+
title="Support reply",
|
|
33
|
+
description="Checks reply length, timeline wording, and refund promises.",
|
|
34
|
+
license="MIT",
|
|
35
|
+
categories=("Support", "Safety"),
|
|
36
|
+
rules=(
|
|
37
|
+
{
|
|
38
|
+
"id": "reply.length",
|
|
39
|
+
"kind": "text.word_count",
|
|
40
|
+
"severity": "error",
|
|
41
|
+
"description": "Reply must contain 40 to 70 words.",
|
|
42
|
+
"config": {"field": "reply", "minimum": 40, "maximum": 70},
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "reply.timeline",
|
|
46
|
+
"kind": "text.required_terms",
|
|
47
|
+
"severity": "error",
|
|
48
|
+
"description": "Reply must explain the timeline.",
|
|
49
|
+
"config": {"field": "reply", "terms": ["timeline"]},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"id": "reply.promise",
|
|
53
|
+
"kind": "text.forbidden_terms",
|
|
54
|
+
"severity": "error",
|
|
55
|
+
"description": "Reply must not promise a guaranteed refund.",
|
|
56
|
+
"config": {"field": "reply", "terms": ["guaranteed refund"]},
|
|
57
|
+
},
|
|
58
|
+
),
|
|
59
|
+
tests={
|
|
60
|
+
"golden": golden,
|
|
61
|
+
"mutation": [
|
|
62
|
+
{
|
|
63
|
+
"id": "forbidden-refund-promise",
|
|
64
|
+
"input": {"reply": "We offer a guaranteed refund."},
|
|
65
|
+
"context": {},
|
|
66
|
+
"expected": {"decision": "deny"},
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
"fail_closed": [
|
|
70
|
+
{
|
|
71
|
+
"id": "missing-reply",
|
|
72
|
+
"input": {},
|
|
73
|
+
"context": {},
|
|
74
|
+
"expected": {"decision": "deny"},
|
|
75
|
+
}
|
|
76
|
+
],
|
|
77
|
+
},
|
|
78
|
+
examples=(
|
|
79
|
+
{
|
|
80
|
+
"name": "delayed-order",
|
|
81
|
+
"input": {"reply": VALID_REPLY},
|
|
82
|
+
"context": {},
|
|
83
|
+
},
|
|
84
|
+
),
|
|
85
|
+
provenance={
|
|
86
|
+
"builder": "rubricloop-sdk",
|
|
87
|
+
"source_repository": "https://rubricloop.com/docs",
|
|
88
|
+
"source_revision": "verify-reply-quick-start",
|
|
89
|
+
"built_at": "",
|
|
90
|
+
},
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def main() -> None:
|
|
95
|
+
parser = argparse.ArgumentParser()
|
|
96
|
+
parser.add_argument("--namespace", required=True)
|
|
97
|
+
parser.add_argument("--version", default="1.0.0")
|
|
98
|
+
parser.add_argument("--output", type=Path, default=Path("dist/support-reply.rlpack"))
|
|
99
|
+
args = parser.parse_args()
|
|
100
|
+
|
|
101
|
+
bundle = build_rlpack(
|
|
102
|
+
manifest(args.namespace, args.version),
|
|
103
|
+
{
|
|
104
|
+
"payload/quick-start.json": json.dumps(
|
|
105
|
+
{
|
|
106
|
+
"example": "sdk/examples/verify_reply.py",
|
|
107
|
+
"model": "gpt-4o-mini",
|
|
108
|
+
},
|
|
109
|
+
sort_keys=True,
|
|
110
|
+
).encode()
|
|
111
|
+
},
|
|
112
|
+
)
|
|
113
|
+
package = read_rlpack(bundle)
|
|
114
|
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
args.output.write_bytes(bundle)
|
|
116
|
+
print(args.output)
|
|
117
|
+
print(package.manifest_digest)
|
|
118
|
+
print(package.archive_digest)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
if __name__ == "__main__":
|
|
122
|
+
main()
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"output": {
|
|
3
|
+
"reply": "We apologize for the delay with your order. Our team is currently investigating the issue and working to resolve it as quickly as possible. You can expect an update on the timeline for your order shortly. Thank you for your patience."
|
|
4
|
+
},
|
|
5
|
+
"context": {}
|
|
6
|
+
}
|
|
@@ -29,6 +29,10 @@ def parser() -> argparse.ArgumentParser:
|
|
|
29
29
|
login.add_argument("--api-key")
|
|
30
30
|
init = commands.add_parser("init")
|
|
31
31
|
init.add_argument("directory", nargs="?", default=".")
|
|
32
|
+
create = commands.add_parser("create")
|
|
33
|
+
create.add_argument("repository")
|
|
34
|
+
create.add_argument("--description", default="")
|
|
35
|
+
create.add_argument("--visibility", choices=("public", "private"), default="private")
|
|
32
36
|
for command in ("validate", "test"):
|
|
33
37
|
item = commands.add_parser(command)
|
|
34
38
|
item.add_argument("package")
|
|
@@ -89,6 +93,20 @@ def execute(args: argparse.Namespace) -> dict | str:
|
|
|
89
93
|
+ "\n"
|
|
90
94
|
)
|
|
91
95
|
return str(target)
|
|
96
|
+
if args.command == "create":
|
|
97
|
+
namespace, slash, name = args.repository.partition("/")
|
|
98
|
+
if not slash or not namespace or not name or "/" in name:
|
|
99
|
+
raise RegistryClientError("repository must be namespace/name")
|
|
100
|
+
return client.json(
|
|
101
|
+
"POST",
|
|
102
|
+
"/v1/registry/repositories",
|
|
103
|
+
payload={
|
|
104
|
+
"namespace": namespace,
|
|
105
|
+
"name": name,
|
|
106
|
+
"description": args.description,
|
|
107
|
+
"visibility": args.visibility,
|
|
108
|
+
},
|
|
109
|
+
)
|
|
92
110
|
if args.command in {"validate", "test"}:
|
|
93
111
|
package = _package(args.package)
|
|
94
112
|
if args.command == "validate":
|
|
@@ -16,7 +16,18 @@ from rubricloop.registry import (
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def test_cli_exposes_complete_lifecycle():
|
|
19
|
-
for command in (
|
|
19
|
+
for command in (
|
|
20
|
+
"login",
|
|
21
|
+
"init",
|
|
22
|
+
"create",
|
|
23
|
+
"validate",
|
|
24
|
+
"test",
|
|
25
|
+
"push",
|
|
26
|
+
"vet",
|
|
27
|
+
"publish",
|
|
28
|
+
"pull",
|
|
29
|
+
"run",
|
|
30
|
+
):
|
|
20
31
|
assert command in parser()._subparsers._group_actions[0].choices
|
|
21
32
|
|
|
22
33
|
|
|
@@ -26,6 +37,42 @@ def test_init_creates_project_without_credentials(tmp_path):
|
|
|
26
37
|
assert json.loads((tmp_path / "rubricloop.json").read_text())["format_version"] == 1
|
|
27
38
|
|
|
28
39
|
|
|
40
|
+
def test_create_calls_repository_endpoint(monkeypatch):
|
|
41
|
+
calls = []
|
|
42
|
+
monkeypatch.setattr(
|
|
43
|
+
RegistryClient,
|
|
44
|
+
"json",
|
|
45
|
+
lambda self, method, path, **kwargs: calls.append((method, path, kwargs)) or {"id": "repo"},
|
|
46
|
+
)
|
|
47
|
+
result = execute(
|
|
48
|
+
parser().parse_args(
|
|
49
|
+
[
|
|
50
|
+
"create",
|
|
51
|
+
"acme/reply-check",
|
|
52
|
+
"--visibility",
|
|
53
|
+
"public",
|
|
54
|
+
"--description",
|
|
55
|
+
"Checks support replies.",
|
|
56
|
+
]
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
assert result == {"id": "repo"}
|
|
60
|
+
assert calls == [
|
|
61
|
+
(
|
|
62
|
+
"POST",
|
|
63
|
+
"/v1/registry/repositories",
|
|
64
|
+
{
|
|
65
|
+
"payload": {
|
|
66
|
+
"namespace": "acme",
|
|
67
|
+
"name": "reply-check",
|
|
68
|
+
"description": "Checks support replies.",
|
|
69
|
+
"visibility": "public",
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
)
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
|
|
29
76
|
def test_credentials_use_keychain_or_explicit_environment(monkeypatch):
|
|
30
77
|
values = {}
|
|
31
78
|
backend_type = type("Backend", (), {})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|