foxy-audit 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foxy_audit-1.0.0/.gitignore +31 -0
- foxy_audit-1.0.0/PKG-INFO +121 -0
- foxy_audit-1.0.0/README.md +87 -0
- foxy_audit-1.0.0/pyproject.toml +49 -0
- foxy_audit-1.0.0/src/foxy_audit/__init__.py +44 -0
- foxy_audit-1.0.0/src/foxy_audit/cli.py +105 -0
- foxy_audit-1.0.0/src/foxy_audit/client.py +124 -0
- foxy_audit-1.0.0/src/foxy_audit/config.py +58 -0
- foxy_audit-1.0.0/src/foxy_audit/dispatch.py +102 -0
- foxy_audit-1.0.0/src/foxy_audit/hashing.py +31 -0
- foxy_audit-1.0.0/src/foxy_audit/pii.py +81 -0
- foxy_audit-1.0.0/src/foxy_audit/udp.py +44 -0
- foxy_audit-1.0.0/tests/test_cli.py +48 -0
- foxy_audit-1.0.0/tests/test_integration.py +147 -0
- foxy_audit-1.0.0/tests/test_pii.py +37 -0
- foxy_audit-1.0.0/tests/test_sdk.py +122 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
|
|
10
|
+
# Virtualenvs
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# Node / Next.js
|
|
16
|
+
node_modules/
|
|
17
|
+
.next/
|
|
18
|
+
*.tsbuildinfo
|
|
19
|
+
|
|
20
|
+
# Secrets / local config
|
|
21
|
+
.env
|
|
22
|
+
.env.local
|
|
23
|
+
backend/.env
|
|
24
|
+
*.local
|
|
25
|
+
|
|
26
|
+
# DB backups (created by deploy/backup.sh on the VM)
|
|
27
|
+
deploy/backups/
|
|
28
|
+
|
|
29
|
+
# OS / editor
|
|
30
|
+
.DS_Store
|
|
31
|
+
Thumbs.db
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: foxy-audit
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Governance-as-Code SDK for AI compliance: local SHA-256 hashing + tamper-evident audit telemetry, with zero raw-text exfiltration.
|
|
5
|
+
Project-URL: Homepage, https://foxyaudit.tech
|
|
6
|
+
Project-URL: Documentation, https://foxyaudit.tech/docs.html
|
|
7
|
+
Project-URL: Source, https://github.com/fatimaatta-09/Foxy-Audit
|
|
8
|
+
Project-URL: Issues, https://github.com/fatimaatta-09/Foxy-Audit/issues
|
|
9
|
+
Author: Foxy Audit
|
|
10
|
+
License: MIT
|
|
11
|
+
Keywords: ai,audit,compliance,eu-ai-act,governance,hipaa,soc2
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: requests<3,>=2.31
|
|
27
|
+
Provides-Extra: pii
|
|
28
|
+
Requires-Dist: presidio-analyzer>=2.2; extra == 'pii'
|
|
29
|
+
Requires-Dist: spacy>=3.7; extra == 'pii'
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: httpx>=0.27; extra == 'test'
|
|
32
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# foxy-audit (SDK)
|
|
36
|
+
|
|
37
|
+
> Governance-as-Code for AI. One decorator → a tamper-evident, privacy-preserving audit trail.
|
|
38
|
+
|
|
39
|
+
The SDK hashes your LLM prompt + response **locally** (SHA-256), throws the raw text away, and
|
|
40
|
+
streams only metadata to the Foxy Audit backend. It also fires a best-effort local UDP ping so the
|
|
41
|
+
desktop "fox" companion reacts in real time (green on every logged call, red on a policy breach).
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install -e . # from this sdk/ folder, for local development
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Runtime dependency: `requests` only.
|
|
50
|
+
|
|
51
|
+
## Use
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import os
|
|
55
|
+
from foxy_audit import FoxyClient
|
|
56
|
+
|
|
57
|
+
foxy = FoxyClient(api_key=os.getenv("FOXY_API_KEY")) # or just rely on the env var
|
|
58
|
+
|
|
59
|
+
@foxy.audit(policy="hipaa_basic")
|
|
60
|
+
def ask_model(prompt: str) -> str:
|
|
61
|
+
return llm_client.generate(prompt) # your existing code — unchanged
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Every call to `ask_model` is now hashed, logged, and graded. Or use the module-level decorator,
|
|
65
|
+
which builds a client from the environment:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from foxy_audit import audit
|
|
69
|
+
|
|
70
|
+
@audit(policy="soc2")
|
|
71
|
+
def summarize(text: str) -> str:
|
|
72
|
+
...
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Attributing the model (`agent`)
|
|
76
|
+
|
|
77
|
+
Pass `agent=` to record *which* model produced the interaction. The backend folds it into the
|
|
78
|
+
tamper-evident hash chain, so the attribution can't be altered after the fact:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
@foxy.audit(policy="soc2", agent="gpt-4o")
|
|
82
|
+
def ask_model(prompt: str) -> str:
|
|
83
|
+
...
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`agent` is optional — rows logged without it hash exactly as before, so existing chains keep
|
|
87
|
+
verifying.
|
|
88
|
+
|
|
89
|
+
## Configuration
|
|
90
|
+
|
|
91
|
+
| Setting | Kwarg | Env var | Default |
|
|
92
|
+
|----------------|----------------|---------------------|--------------------------|
|
|
93
|
+
| API key | `api_key` | `FOXY_API_KEY` | _(none → HTTP disabled)_ |
|
|
94
|
+
| Backend URL | `endpoint` | `FOXY_BACKEND_URL` | `http://127.0.0.1:8000` |
|
|
95
|
+
| Desktop ping | `desktop_ping` | — | `True` (127.0.0.1:9999) |
|
|
96
|
+
|
|
97
|
+
With no API key the SDK is a **graceful no-op for the cloud path**: it still runs your function and
|
|
98
|
+
still pings the desktop fox, but skips the HTTP upload — so you can see the fox react before any
|
|
99
|
+
backend exists.
|
|
100
|
+
|
|
101
|
+
## Guarantees
|
|
102
|
+
|
|
103
|
+
- **Never blocks** your function — the HTTP upload runs on a background daemon thread.
|
|
104
|
+
- **Never raises** its own errors into your app — all telemetry failures are swallowed.
|
|
105
|
+
- **Never sees raw text** server-side — only SHA-256 digests + token count + policy tag leave the host.
|
|
106
|
+
- Works with both **sync and async** functions; the return value is always passed through unchanged.
|
|
107
|
+
|
|
108
|
+
## What gets sent
|
|
109
|
+
|
|
110
|
+
To the backend (`POST /v1/logs`, `Authorization: Bearer <key>`):
|
|
111
|
+
|
|
112
|
+
```json
|
|
113
|
+
{"prompt_hash": "<64 hex>", "response_hash": "<64 hex>", "token_count": 123, "policy_tag": "hipaa_basic"}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
To the desktop fox (UDP `127.0.0.1:9999`):
|
|
117
|
+
|
|
118
|
+
```json
|
|
119
|
+
{"event": "hash_ok", "policy": "hipaa_basic", "tokens": 123, "ts": 1719300000}
|
|
120
|
+
{"event": "policy_breach", "reason": "...", "risk_score": 87, "policy": "hipaa_basic", "ts": ...}
|
|
121
|
+
```
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# foxy-audit (SDK)
|
|
2
|
+
|
|
3
|
+
> Governance-as-Code for AI. One decorator → a tamper-evident, privacy-preserving audit trail.
|
|
4
|
+
|
|
5
|
+
The SDK hashes your LLM prompt + response **locally** (SHA-256), throws the raw text away, and
|
|
6
|
+
streams only metadata to the Foxy Audit backend. It also fires a best-effort local UDP ping so the
|
|
7
|
+
desktop "fox" companion reacts in real time (green on every logged call, red on a policy breach).
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install -e . # from this sdk/ folder, for local development
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Runtime dependency: `requests` only.
|
|
16
|
+
|
|
17
|
+
## Use
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import os
|
|
21
|
+
from foxy_audit import FoxyClient
|
|
22
|
+
|
|
23
|
+
foxy = FoxyClient(api_key=os.getenv("FOXY_API_KEY")) # or just rely on the env var
|
|
24
|
+
|
|
25
|
+
@foxy.audit(policy="hipaa_basic")
|
|
26
|
+
def ask_model(prompt: str) -> str:
|
|
27
|
+
return llm_client.generate(prompt) # your existing code — unchanged
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Every call to `ask_model` is now hashed, logged, and graded. Or use the module-level decorator,
|
|
31
|
+
which builds a client from the environment:
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from foxy_audit import audit
|
|
35
|
+
|
|
36
|
+
@audit(policy="soc2")
|
|
37
|
+
def summarize(text: str) -> str:
|
|
38
|
+
...
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### Attributing the model (`agent`)
|
|
42
|
+
|
|
43
|
+
Pass `agent=` to record *which* model produced the interaction. The backend folds it into the
|
|
44
|
+
tamper-evident hash chain, so the attribution can't be altered after the fact:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
@foxy.audit(policy="soc2", agent="gpt-4o")
|
|
48
|
+
def ask_model(prompt: str) -> str:
|
|
49
|
+
...
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`agent` is optional — rows logged without it hash exactly as before, so existing chains keep
|
|
53
|
+
verifying.
|
|
54
|
+
|
|
55
|
+
## Configuration
|
|
56
|
+
|
|
57
|
+
| Setting | Kwarg | Env var | Default |
|
|
58
|
+
|----------------|----------------|---------------------|--------------------------|
|
|
59
|
+
| API key | `api_key` | `FOXY_API_KEY` | _(none → HTTP disabled)_ |
|
|
60
|
+
| Backend URL | `endpoint` | `FOXY_BACKEND_URL` | `http://127.0.0.1:8000` |
|
|
61
|
+
| Desktop ping | `desktop_ping` | — | `True` (127.0.0.1:9999) |
|
|
62
|
+
|
|
63
|
+
With no API key the SDK is a **graceful no-op for the cloud path**: it still runs your function and
|
|
64
|
+
still pings the desktop fox, but skips the HTTP upload — so you can see the fox react before any
|
|
65
|
+
backend exists.
|
|
66
|
+
|
|
67
|
+
## Guarantees
|
|
68
|
+
|
|
69
|
+
- **Never blocks** your function — the HTTP upload runs on a background daemon thread.
|
|
70
|
+
- **Never raises** its own errors into your app — all telemetry failures are swallowed.
|
|
71
|
+
- **Never sees raw text** server-side — only SHA-256 digests + token count + policy tag leave the host.
|
|
72
|
+
- Works with both **sync and async** functions; the return value is always passed through unchanged.
|
|
73
|
+
|
|
74
|
+
## What gets sent
|
|
75
|
+
|
|
76
|
+
To the backend (`POST /v1/logs`, `Authorization: Bearer <key>`):
|
|
77
|
+
|
|
78
|
+
```json
|
|
79
|
+
{"prompt_hash": "<64 hex>", "response_hash": "<64 hex>", "token_count": 123, "policy_tag": "hipaa_basic"}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
To the desktop fox (UDP `127.0.0.1:9999`):
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{"event": "hash_ok", "policy": "hipaa_basic", "tokens": 123, "ts": 1719300000}
|
|
86
|
+
{"event": "policy_breach", "reason": "...", "risk_score": 87, "policy": "hipaa_basic", "ts": ...}
|
|
87
|
+
```
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "foxy-audit"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Governance-as-Code SDK for AI compliance: local SHA-256 hashing + tamper-evident audit telemetry, with zero raw-text exfiltration."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Foxy Audit" }]
|
|
13
|
+
keywords = ["compliance", "audit", "ai", "governance", "hipaa", "soc2", "eu-ai-act"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 5 - Production/Stable",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.9",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Security",
|
|
26
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
# A library: bound the major (never exact-pin) so it stays co-installable in a
|
|
30
|
+
# consumer's environment. (Phase 5 · 5F)
|
|
31
|
+
dependencies = ["requests>=2.31,<3"]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://foxyaudit.tech"
|
|
35
|
+
Documentation = "https://foxyaudit.tech/docs.html"
|
|
36
|
+
Source = "https://github.com/fatimaatta-09/Foxy-Audit"
|
|
37
|
+
Issues = "https://github.com/fatimaatta-09/Foxy-Audit/issues"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
foxy = "foxy_audit.cli:main"
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
test = ["httpx>=0.27", "pytest>=8"]
|
|
44
|
+
# Deep NLP PII detection (names / locations / MRNs). Also needs a spaCy model:
|
|
45
|
+
# python -m spacy download en_core_web_lg
|
|
46
|
+
pii = ["presidio-analyzer>=2.2", "spacy>=3.7"]
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/foxy_audit"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Foxy Audit SDK — Governance-as-Code for AI applications.
|
|
3
|
+
|
|
4
|
+
Wrap any LLM call with the ``@audit`` decorator and every invocation is
|
|
5
|
+
hashed locally (SHA-256), the raw text is discarded, and only metadata is
|
|
6
|
+
streamed to the Foxy Audit backend — plus a best-effort local UDP ping that
|
|
7
|
+
makes the desktop "fox" companion react in real time.
|
|
8
|
+
|
|
9
|
+
Quickstart
|
|
10
|
+
----------
|
|
11
|
+
from foxy_audit import FoxyClient
|
|
12
|
+
|
|
13
|
+
foxy = FoxyClient(api_key="foxy_sk_...") # or set $FOXY_API_KEY
|
|
14
|
+
|
|
15
|
+
@foxy.audit(policy="hipaa_basic")
|
|
16
|
+
def ask_model(prompt: str) -> str:
|
|
17
|
+
return llm_client.generate(prompt) # your code, unchanged
|
|
18
|
+
|
|
19
|
+
The decorator never blocks the wrapped function and never raises its own
|
|
20
|
+
errors into your application — telemetry failure is swallowed silently.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from .client import FoxyClient
|
|
24
|
+
from .config import FoxyConfig
|
|
25
|
+
|
|
26
|
+
__version__ = "0.1.0"
|
|
27
|
+
__all__ = ["FoxyClient", "FoxyConfig", "audit", "__version__"]
|
|
28
|
+
|
|
29
|
+
# Module-level convenience: a lazily-created client configured from the
|
|
30
|
+
# environment (FOXY_API_KEY / FOXY_BACKEND_URL), so `from foxy_audit import audit`
|
|
31
|
+
# works without explicitly constructing a client.
|
|
32
|
+
_default_client = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _client() -> FoxyClient:
|
|
36
|
+
global _default_client
|
|
37
|
+
if _default_client is None:
|
|
38
|
+
_default_client = FoxyClient()
|
|
39
|
+
return _default_client
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def audit(policy: str = "default"):
|
|
43
|
+
"""Decorator bound to the default environment-configured client."""
|
|
44
|
+
return _client().audit(policy)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""foxy doctor — one-command, end-to-end connectivity check for the Foxy Audit SDK.
|
|
2
|
+
|
|
3
|
+
Sends a real test interaction through the whole path and reports what is actually
|
|
4
|
+
connected: the backend (authenticated ingest), the desktop pet (a local UDP signal),
|
|
5
|
+
and the tamper-evident chain (the test log verified server-side). This is the
|
|
6
|
+
highest-leverage "is it all wired together?" check for you or a judge.
|
|
7
|
+
|
|
8
|
+
foxy doctor
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from . import hashing, udp
|
|
18
|
+
from .config import FoxyConfig
|
|
19
|
+
|
|
20
|
+
def _pick(uni: str, plain: str) -> str:
|
|
21
|
+
"""Emoji if the terminal can encode it, else an ASCII marker (Windows cp1252-safe)."""
|
|
22
|
+
try:
|
|
23
|
+
uni.encode(sys.stdout.encoding or "utf-8")
|
|
24
|
+
return uni
|
|
25
|
+
except (UnicodeEncodeError, LookupError, TypeError):
|
|
26
|
+
return plain
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
_OK = _pick("✅", "[OK]")
|
|
30
|
+
_X = _pick("❌", "[X]")
|
|
31
|
+
_ARROW = _pick("→", "->")
|
|
32
|
+
_FOX = _pick("🦊", "")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _doctor(cfg: FoxyConfig) -> int:
|
|
36
|
+
print("Foxy Audit — doctor")
|
|
37
|
+
print(f" backend : {cfg.endpoint}")
|
|
38
|
+
print(f" pet : {cfg.udp_host}:{cfg.udp_port}")
|
|
39
|
+
if not cfg.api_key:
|
|
40
|
+
print(f" {_X} No API key set. Export FOXY_API_KEY (get one from a free signup or the "
|
|
41
|
+
f"dashboard), then re-run `foxy doctor`.")
|
|
42
|
+
return 1
|
|
43
|
+
|
|
44
|
+
# 1) BACKEND — authenticated ingest of one real test interaction.
|
|
45
|
+
prompt, response = "foxy doctor test prompt", "foxy doctor test response"
|
|
46
|
+
payload = [{
|
|
47
|
+
"prompt_hash": hashing.sha256_hex(prompt),
|
|
48
|
+
"response_hash": hashing.sha256_hex(response),
|
|
49
|
+
"token_count": hashing.estimate_tokens(prompt, response),
|
|
50
|
+
"policy_tag": "foxy-doctor",
|
|
51
|
+
}]
|
|
52
|
+
try:
|
|
53
|
+
r = requests.post(f"{cfg.endpoint}/v1/logs/batch",
|
|
54
|
+
headers={"Authorization": f"Bearer {cfg.api_key}"},
|
|
55
|
+
json=payload, timeout=cfg.timeout)
|
|
56
|
+
except requests.RequestException as exc:
|
|
57
|
+
print(f" {_X} Backend unreachable at {cfg.endpoint} ({exc}). Is it running / is the URL right?")
|
|
58
|
+
return 1
|
|
59
|
+
if r.status_code == 401:
|
|
60
|
+
print(f" {_X} Backend rejected the API key (401). Check FOXY_API_KEY.")
|
|
61
|
+
return 1
|
|
62
|
+
if r.status_code >= 400:
|
|
63
|
+
print(f" {_X} Backend error {r.status_code}: {r.text[:200]}")
|
|
64
|
+
return 1
|
|
65
|
+
print(f" {_OK} Backend reachable + authenticated — test log accepted (HTTP {r.status_code}).")
|
|
66
|
+
|
|
67
|
+
# 2) DESKTOP PET — fire the same instant signal the SDK sends on every call.
|
|
68
|
+
sent = udp.send_ping({"event": "evaluating", "policy": "foxy-doctor"},
|
|
69
|
+
cfg.udp_host, cfg.udp_port)
|
|
70
|
+
tail = "watch the fox react" if sent else "no pet is listening — that is fine"
|
|
71
|
+
print(f" {_ARROW} Sent a signal to the desktop pet ({tail}).")
|
|
72
|
+
|
|
73
|
+
# 3) CHAIN — confirm the ledger verifies server-side.
|
|
74
|
+
try:
|
|
75
|
+
v = requests.get(f"{cfg.endpoint}/v1/verify",
|
|
76
|
+
headers={"Authorization": f"Bearer {cfg.api_key}"},
|
|
77
|
+
timeout=cfg.timeout)
|
|
78
|
+
data = v.json() if v.ok else {}
|
|
79
|
+
except requests.RequestException:
|
|
80
|
+
data = {}
|
|
81
|
+
if data.get("ok"):
|
|
82
|
+
print(f" {_OK} Ledger verified — {data.get('count', 0)} entries, chain intact.")
|
|
83
|
+
else:
|
|
84
|
+
print(f" {_OK} Test log written. (Grading + chain verification complete server-side "
|
|
85
|
+
f"within a few seconds.)")
|
|
86
|
+
|
|
87
|
+
print(f"\nAll set — your calls are flowing end-to-end. {_FOX}".rstrip())
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main(argv=None) -> int:
|
|
92
|
+
parser = argparse.ArgumentParser(prog="foxy", description="Foxy Audit SDK tools.")
|
|
93
|
+
sub = parser.add_subparsers(dest="command")
|
|
94
|
+
doc = sub.add_parser("doctor", help="check backend + desktop pet + chain end-to-end")
|
|
95
|
+
doc.add_argument("--api-key", default=None, help="override FOXY_API_KEY")
|
|
96
|
+
doc.add_argument("--backend", default=None, help="override FOXY_BACKEND_URL")
|
|
97
|
+
args = parser.parse_args(argv)
|
|
98
|
+
if args.command == "doctor":
|
|
99
|
+
return _doctor(FoxyConfig.resolve(api_key=args.api_key, endpoint=args.backend))
|
|
100
|
+
parser.print_help()
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
sys.exit(main())
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""FoxyClient — the developer-facing decorator.
|
|
2
|
+
|
|
3
|
+
`@client.audit(policy)` wraps any function (sync or async) that calls an LLM.
|
|
4
|
+
After the wrapped function returns, the SDK:
|
|
5
|
+
|
|
6
|
+
1. SHA-256-hashes the prompt and response locally, then discards the raw text.
|
|
7
|
+
2. Fires an instant best-effort `hash_ok` UDP ping to the desktop fox.
|
|
8
|
+
3. Enqueues the metadata for background HTTP delivery to the backend (only
|
|
9
|
+
when an API key is configured).
|
|
10
|
+
|
|
11
|
+
The wrapped function's own return value is always passed through unchanged, and
|
|
12
|
+
the SDK's own bookkeeping can never raise into the host application.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import functools
|
|
18
|
+
import inspect
|
|
19
|
+
import logging
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
from . import dispatch, hashing, pii, udp
|
|
23
|
+
from .config import FoxyConfig
|
|
24
|
+
|
|
25
|
+
log = logging.getLogger("foxy_audit")
|
|
26
|
+
|
|
27
|
+
_POLICY_RE = re.compile(r"^[a-z0-9_]{1,32}$")
|
|
28
|
+
_PROMPT_KWARGS = ("prompt", "user_prompt", "message", "text", "input", "query")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _extract_prompt(args: tuple, kwargs: dict) -> str:
|
|
32
|
+
"""Best-effort: the first known prompt kwarg, else the first string arg."""
|
|
33
|
+
for key in _PROMPT_KWARGS:
|
|
34
|
+
val = kwargs.get(key)
|
|
35
|
+
if isinstance(val, str):
|
|
36
|
+
return val
|
|
37
|
+
for arg in args:
|
|
38
|
+
if isinstance(arg, str):
|
|
39
|
+
return arg
|
|
40
|
+
return ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class FoxyClient:
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
api_key: str | None = None,
|
|
47
|
+
endpoint: str | None = None,
|
|
48
|
+
udp_host: str | None = None,
|
|
49
|
+
udp_port: int | None = None,
|
|
50
|
+
desktop_ping: bool = True,
|
|
51
|
+
timeout: float = 5.0,
|
|
52
|
+
) -> None:
|
|
53
|
+
self.cfg = FoxyConfig.resolve(
|
|
54
|
+
api_key=api_key,
|
|
55
|
+
endpoint=endpoint,
|
|
56
|
+
udp_host=udp_host,
|
|
57
|
+
udp_port=udp_port,
|
|
58
|
+
desktop_ping=desktop_ping,
|
|
59
|
+
timeout=timeout,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def enabled(self) -> bool:
|
|
64
|
+
return self.cfg.enabled
|
|
65
|
+
|
|
66
|
+
def audit(self, policy: str = "default", agent: str | None = None):
|
|
67
|
+
"""Return a decorator that audits the wrapped LLM-calling function.
|
|
68
|
+
|
|
69
|
+
`agent` records which model/agent produced the interaction (e.g.
|
|
70
|
+
"gpt-4o", "claude-3-opus"); the backend folds it into the tamper-evident
|
|
71
|
+
hash chain so it can't be altered after the fact (6B)."""
|
|
72
|
+
if not _POLICY_RE.match(policy):
|
|
73
|
+
log.warning("foxy-audit: invalid policy tag %r; falling back to 'default'", policy)
|
|
74
|
+
policy = "default"
|
|
75
|
+
|
|
76
|
+
def decorator(fn):
|
|
77
|
+
if inspect.iscoroutinefunction(fn):
|
|
78
|
+
@functools.wraps(fn)
|
|
79
|
+
async def awrapper(*args, **kwargs):
|
|
80
|
+
response = await fn(*args, **kwargs)
|
|
81
|
+
self.log_interaction(_extract_prompt(args, kwargs), response, policy, agent)
|
|
82
|
+
return response
|
|
83
|
+
return awrapper
|
|
84
|
+
|
|
85
|
+
@functools.wraps(fn)
|
|
86
|
+
def wrapper(*args, **kwargs):
|
|
87
|
+
response = fn(*args, **kwargs)
|
|
88
|
+
self.log_interaction(_extract_prompt(args, kwargs), response, policy, agent)
|
|
89
|
+
return response
|
|
90
|
+
return wrapper
|
|
91
|
+
|
|
92
|
+
return decorator
|
|
93
|
+
|
|
94
|
+
# ── internal ──────────────────────────────────────────────────────────
|
|
95
|
+
def log_interaction(self, prompt, response, policy: str, agent: str | None = None) -> None:
|
|
96
|
+
"""Perform cryptographic hashing synchronously and push to AsyncDispatcher."""
|
|
97
|
+
try:
|
|
98
|
+
prompt_s = "" if prompt is None else str(prompt)
|
|
99
|
+
response_s = "" if response is None else str(response)
|
|
100
|
+
|
|
101
|
+
# The backend owns the tamper-evident hash chain (it re-derives each
|
|
102
|
+
# link server-side), so the SDK only ships the per-interaction hashes;
|
|
103
|
+
# a client-side chain_hash/timestamp would just be ignored.
|
|
104
|
+
payload = {
|
|
105
|
+
"prompt_hash": hashing.sha256_hex(prompt_s),
|
|
106
|
+
"response_hash": hashing.sha256_hex(response_s),
|
|
107
|
+
"token_count": hashing.estimate_tokens(prompt_s, response_s),
|
|
108
|
+
"policy_tag": policy,
|
|
109
|
+
"pii_signals": pii.detect_pii(prompt_s, response_s),
|
|
110
|
+
}
|
|
111
|
+
if agent:
|
|
112
|
+
payload["agent"] = agent
|
|
113
|
+
# raw text goes out of scope here — never stored or transmitted
|
|
114
|
+
|
|
115
|
+
if self.cfg.desktop_ping:
|
|
116
|
+
udp.send_ping(
|
|
117
|
+
{"event": "evaluating", "policy": policy, "tokens": payload["token_count"]},
|
|
118
|
+
self.cfg.udp_host,
|
|
119
|
+
self.cfg.udp_port,
|
|
120
|
+
)
|
|
121
|
+
if self.cfg.enabled:
|
|
122
|
+
dispatch.submit(self.cfg, payload)
|
|
123
|
+
except Exception as exc: # telemetry must never break the host app
|
|
124
|
+
log.debug("foxy-audit observe error: %s", exc)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Configuration resolution for the Foxy Audit SDK.
|
|
2
|
+
|
|
3
|
+
Priority for every setting: explicit kwarg → environment variable → default.
|
|
4
|
+
The SDK is a graceful no-op for the HTTP path when no API key is configured
|
|
5
|
+
(it still fires the local UDP ping so the desktop fox reacts), so importing
|
|
6
|
+
and decorating is always safe even before a key/backend exist.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
# The UDP host/port are fixed by the desktop app's sdk_bridge listener.
|
|
15
|
+
DEFAULT_ENDPOINT = "http://127.0.0.1:8000"
|
|
16
|
+
DEFAULT_UDP_HOST = "127.0.0.1"
|
|
17
|
+
DEFAULT_UDP_PORT = 9999
|
|
18
|
+
DEFAULT_TIMEOUT = 5.0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class FoxyConfig:
|
|
23
|
+
api_key: str = ""
|
|
24
|
+
endpoint: str = DEFAULT_ENDPOINT
|
|
25
|
+
udp_host: str = DEFAULT_UDP_HOST
|
|
26
|
+
udp_port: int = DEFAULT_UDP_PORT
|
|
27
|
+
desktop_ping: bool = True
|
|
28
|
+
timeout: float = DEFAULT_TIMEOUT
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def resolve(
|
|
32
|
+
cls,
|
|
33
|
+
api_key: str | None = None,
|
|
34
|
+
endpoint: str | None = None,
|
|
35
|
+
udp_host: str | None = None,
|
|
36
|
+
udp_port: int | None = None,
|
|
37
|
+
desktop_ping: bool = True,
|
|
38
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
39
|
+
) -> "FoxyConfig":
|
|
40
|
+
key = api_key if api_key is not None else os.getenv("FOXY_API_KEY", "")
|
|
41
|
+
ep = endpoint if endpoint is not None else os.getenv("FOXY_BACKEND_URL", DEFAULT_ENDPOINT)
|
|
42
|
+
return cls(
|
|
43
|
+
api_key=key.strip(),
|
|
44
|
+
endpoint=ep.rstrip("/"),
|
|
45
|
+
udp_host=udp_host or DEFAULT_UDP_HOST,
|
|
46
|
+
udp_port=int(udp_port or DEFAULT_UDP_PORT),
|
|
47
|
+
desktop_ping=desktop_ping,
|
|
48
|
+
timeout=timeout,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def enabled(self) -> bool:
|
|
53
|
+
"""True when the SDK should stream to the cloud backend.
|
|
54
|
+
|
|
55
|
+
When False, the decorator still runs the wrapped function and still
|
|
56
|
+
fires the local desktop ping — it just skips the HTTP POST.
|
|
57
|
+
"""
|
|
58
|
+
return bool(self.api_key)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Background dispatch — keeps all network I/O off the host application's thread.
|
|
2
|
+
|
|
3
|
+
A single lazily-started daemon thread drains a queue of ingest jobs in batches.
|
|
4
|
+
For each batch, it POSTs the metadata to the backend.
|
|
5
|
+
|
|
6
|
+
Every failure path is swallowed and logged at debug level — telemetry must never
|
|
7
|
+
crash, block, or surface an error to the host app.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import atexit
|
|
13
|
+
import logging
|
|
14
|
+
import queue
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
from .config import FoxyConfig
|
|
20
|
+
|
|
21
|
+
log = logging.getLogger("foxy_audit")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AsyncDispatcher:
|
|
25
|
+
def __init__(self, batch_size: int = 10, flush_interval: float = 1.0) -> None:
|
|
26
|
+
self._q: "queue.Queue[tuple[FoxyConfig, dict]]" = queue.Queue()
|
|
27
|
+
self._thread: threading.Thread | None = None
|
|
28
|
+
self._lock = threading.Lock()
|
|
29
|
+
self.batch_size = batch_size
|
|
30
|
+
self.flush_interval = flush_interval
|
|
31
|
+
self._shutdown = False
|
|
32
|
+
atexit.register(self.flush)
|
|
33
|
+
|
|
34
|
+
def _ensure_worker(self) -> None:
|
|
35
|
+
if self._thread is not None and self._thread.is_alive():
|
|
36
|
+
return
|
|
37
|
+
with self._lock:
|
|
38
|
+
if self._thread is None or not self._thread.is_alive():
|
|
39
|
+
self._shutdown = False
|
|
40
|
+
self._thread = threading.Thread(
|
|
41
|
+
target=self._run, name="foxy-audit-async-dispatch", daemon=True
|
|
42
|
+
)
|
|
43
|
+
self._thread.start()
|
|
44
|
+
|
|
45
|
+
def submit(self, cfg: FoxyConfig, payload: dict) -> None:
|
|
46
|
+
self._ensure_worker()
|
|
47
|
+
self._q.put((cfg, payload))
|
|
48
|
+
|
|
49
|
+
def _run(self) -> None:
|
|
50
|
+
batch = []
|
|
51
|
+
last_flush = time.time()
|
|
52
|
+
|
|
53
|
+
while not self._shutdown or not self._q.empty():
|
|
54
|
+
try:
|
|
55
|
+
# Wait up to the flush interval for an item
|
|
56
|
+
timeout = max(0.1, self.flush_interval - (time.time() - last_flush))
|
|
57
|
+
cfg, payload = self._q.get(timeout=timeout)
|
|
58
|
+
batch.append((cfg, payload))
|
|
59
|
+
except queue.Empty:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
now = time.time()
|
|
63
|
+
if batch and (len(batch) >= self.batch_size or now - last_flush >= self.flush_interval):
|
|
64
|
+
self._process_batch(batch)
|
|
65
|
+
batch = []
|
|
66
|
+
last_flush = now
|
|
67
|
+
|
|
68
|
+
def _process_batch(self, batch: list[tuple[FoxyConfig, dict]]) -> None:
|
|
69
|
+
if not batch:
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
cfg = batch[0][0]
|
|
73
|
+
payloads = [item[1] for item in batch]
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
headers = {"Authorization": f"Bearer {cfg.api_key}", "Content-Type": "application/json"}
|
|
77
|
+
|
|
78
|
+
# Normalise the endpoint to always be .../v1/logs/batch
|
|
79
|
+
# Strip any trailing slashes and known suffixes, then append the canonical path.
|
|
80
|
+
base = cfg.endpoint.rstrip("/")
|
|
81
|
+
for suffix in ("/v1/logs/batch", "/v1/logs", "/logs/batch", "/logs"):
|
|
82
|
+
if base.endswith(suffix):
|
|
83
|
+
base = base[: -len(suffix)]
|
|
84
|
+
break
|
|
85
|
+
endpoint = f"{base}/v1/logs/batch"
|
|
86
|
+
|
|
87
|
+
resp = requests.post(endpoint, json=payloads, headers=headers, timeout=cfg.timeout)
|
|
88
|
+
resp.raise_for_status()
|
|
89
|
+
except Exception as exc:
|
|
90
|
+
log.debug("foxy-audit POST failed: %s", exc)
|
|
91
|
+
|
|
92
|
+
def flush(self) -> None:
|
|
93
|
+
self._shutdown = True
|
|
94
|
+
if self._thread and self._thread.is_alive():
|
|
95
|
+
self._thread.join(timeout=2.0)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
_DISPATCHER = AsyncDispatcher()
|
|
99
|
+
|
|
100
|
+
def submit(cfg: FoxyConfig, payload: dict) -> None:
|
|
101
|
+
"""Enqueue an ingest job for background delivery."""
|
|
102
|
+
_DISPATCHER.submit(cfg, payload)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Local hashing — the heart of the zero-knowledge ("data blindness") design.
|
|
2
|
+
|
|
3
|
+
Prompt and response are hashed here, in-process, and the raw strings are never
|
|
4
|
+
returned, stored, or transmitted. Only the resulting digests + a token estimate
|
|
5
|
+
leave this module.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sha256_hex(text: str) -> str:
|
|
14
|
+
"""Lowercase 64-char hex SHA-256 of the UTF-8 encoding of `text`."""
|
|
15
|
+
return hashlib.sha256(str(text).encode("utf-8")).hexdigest()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def estimate_tokens(*parts: str) -> int:
|
|
19
|
+
"""Cheap, dependency-free token estimate.
|
|
20
|
+
|
|
21
|
+
Approximates a tokenizer with max(chars/4, whitespace-word-count) per part —
|
|
22
|
+
good enough for anomaly detection on the backend without pulling in a heavy
|
|
23
|
+
tokenizer dependency. Swap for tiktoken later if exactness is needed.
|
|
24
|
+
"""
|
|
25
|
+
total = 0
|
|
26
|
+
for part in parts:
|
|
27
|
+
s = str(part)
|
|
28
|
+
if not s:
|
|
29
|
+
continue
|
|
30
|
+
total += max(len(s) // 4, len(s.split()))
|
|
31
|
+
return total
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Client-side PII detection (Phase 5 · 5J).
|
|
2
|
+
|
|
3
|
+
Runs in the SDK — the ONLY place that ever sees raw prompt/response text (the
|
|
4
|
+
backend receives hashes + signals only). A lightweight, dependency-free regex
|
|
5
|
+
layer runs ALWAYS; Microsoft Presidio (deep NLP: names, locations, MRNs) is used
|
|
6
|
+
ONLY when the optional extra is installed (`pip install foxy-audit[pii]`), so the
|
|
7
|
+
base SDK stays tiny. Detection emits SIGNAL LABELS (never the raw values), which
|
|
8
|
+
ride along in the audit metadata for the judge to weigh.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
_EMAIL_RE = re.compile(r"[\w.\-]+@[\w\-]+\.\w+")
|
|
16
|
+
_SSN_RE = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
|
|
17
|
+
_PHONE_RE = re.compile(r"(?<!\d)(?:\+?\d{1,3}[ .\-]?)?\(?\d{3}\)?[ .\-]?\d{3}[ .\-]?\d{4}(?!\d)")
|
|
18
|
+
_IPV4_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
|
|
19
|
+
_CARD_CANDIDATE_RE = re.compile(r"(?<!\d)(?:\d[ \-]?){13,19}(?!\d)")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _luhn_ok(digits: str) -> bool:
|
|
23
|
+
"""Standard Luhn checksum — separates real card numbers from any 13–19 digit run."""
|
|
24
|
+
if not (13 <= len(digits) <= 19):
|
|
25
|
+
return False
|
|
26
|
+
total = 0
|
|
27
|
+
for i, ch in enumerate(reversed(digits)):
|
|
28
|
+
n = int(ch)
|
|
29
|
+
if i % 2 == 1: # double every second digit from the right
|
|
30
|
+
n *= 2
|
|
31
|
+
if n > 9:
|
|
32
|
+
n -= 9
|
|
33
|
+
total += n
|
|
34
|
+
return total % 10 == 0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _has_card(text: str) -> bool:
|
|
38
|
+
for m in _CARD_CANDIDATE_RE.finditer(text):
|
|
39
|
+
if _luhn_ok(re.sub(r"\D", "", m.group())):
|
|
40
|
+
return True
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# ── optional Presidio (deep NLP) — only if the [pii] extra is installed ───────
|
|
45
|
+
_PRESIDIO = None # None = not tried yet, False = unavailable, else the engine
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _presidio_signals(text: str) -> list[str]:
|
|
49
|
+
global _PRESIDIO
|
|
50
|
+
if _PRESIDIO is None:
|
|
51
|
+
try:
|
|
52
|
+
from presidio_analyzer import AnalyzerEngine
|
|
53
|
+
_PRESIDIO = AnalyzerEngine()
|
|
54
|
+
except Exception: # not installed / model missing → never retry
|
|
55
|
+
_PRESIDIO = False
|
|
56
|
+
if not _PRESIDIO:
|
|
57
|
+
return []
|
|
58
|
+
try:
|
|
59
|
+
results = _PRESIDIO.analyze(text=text, language="en")
|
|
60
|
+
return [f"presidio:{r.entity_type.lower()}" for r in results if r.score >= 0.5]
|
|
61
|
+
except Exception:
|
|
62
|
+
return []
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def detect_pii(prompt_s: str, response_s: str) -> list[str]:
|
|
66
|
+
"""De-duplicated PII SIGNAL labels (never raw values) for prompt+response.
|
|
67
|
+
Lightweight regex always; Presidio signals appended when the extra is present."""
|
|
68
|
+
combined = f"{prompt_s} {response_s}"
|
|
69
|
+
signals: list[str] = []
|
|
70
|
+
if _EMAIL_RE.search(combined):
|
|
71
|
+
signals.append("email")
|
|
72
|
+
if _SSN_RE.search(combined):
|
|
73
|
+
signals.append("ssn_pattern")
|
|
74
|
+
if _PHONE_RE.search(combined):
|
|
75
|
+
signals.append("phone")
|
|
76
|
+
if _IPV4_RE.search(combined):
|
|
77
|
+
signals.append("ip_address")
|
|
78
|
+
if _has_card(combined):
|
|
79
|
+
signals.append("credit_card")
|
|
80
|
+
signals.extend(_presidio_signals(combined))
|
|
81
|
+
return sorted(set(signals))
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Best-effort local UDP ping to the desktop fox companion.
|
|
2
|
+
|
|
3
|
+
Speaks the exact protocol of the desktop app's sdk_bridge listener:
|
|
4
|
+
a JSON datagram (<= 4096 bytes) sent to 127.0.0.1:9999, with an `event` of
|
|
5
|
+
"evaluating" (grading pending — fired instantly), "policy_breach", or the legacy
|
|
6
|
+
"hash_ok". This is purely cosmetic ambient feedback (the fox reacts); the
|
|
7
|
+
authoritative record is the HTTP-written
|
|
8
|
+
ledger row, so packet loss here is harmless. The whole send is wrapped so it
|
|
9
|
+
can never raise into the SDK or the host application.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import socket
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
MAX_PACKET = 4096 # must match sdk_bridge.MAX_PACKET_SIZE
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def send_ping(payload: dict, host: str = "127.0.0.1", port: int = 9999) -> bool:
|
|
22
|
+
"""Fire a single UDP datagram. Returns True on success, False otherwise.
|
|
23
|
+
|
|
24
|
+
Always stamps `ts`. If the encoded payload exceeds the listener's 4096-byte
|
|
25
|
+
limit, the `reason` field (if present) is trimmed before re-encoding.
|
|
26
|
+
"""
|
|
27
|
+
try:
|
|
28
|
+
msg = dict(payload)
|
|
29
|
+
msg.setdefault("ts", time.time())
|
|
30
|
+
data = json.dumps(msg).encode("utf-8")
|
|
31
|
+
if len(data) > MAX_PACKET and isinstance(msg.get("reason"), str):
|
|
32
|
+
overflow = len(data) - MAX_PACKET
|
|
33
|
+
msg["reason"] = msg["reason"][: max(0, len(msg["reason"]) - overflow - 1)]
|
|
34
|
+
data = json.dumps(msg).encode("utf-8")
|
|
35
|
+
if len(data) > MAX_PACKET:
|
|
36
|
+
return False # still too big (no reason field to trim) — drop it
|
|
37
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
38
|
+
try:
|
|
39
|
+
sock.sendto(data, (host, port))
|
|
40
|
+
finally:
|
|
41
|
+
sock.close()
|
|
42
|
+
return True
|
|
43
|
+
except Exception:
|
|
44
|
+
return False
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""`foxy doctor` CLI — backend / pet / chain checks, with the network mocked."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
import foxy_audit.cli as cli
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class _Resp:
|
|
10
|
+
def __init__(self, status=202, payload=None, text=""):
|
|
11
|
+
self.status_code = status
|
|
12
|
+
self._payload = payload or {}
|
|
13
|
+
self.text = text
|
|
14
|
+
|
|
15
|
+
@property
|
|
16
|
+
def ok(self):
|
|
17
|
+
return self.status_code < 400
|
|
18
|
+
|
|
19
|
+
def json(self):
|
|
20
|
+
return self._payload
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_doctor_all_green(monkeypatch, capsys):
|
|
24
|
+
monkeypatch.setattr(cli.requests, "post", lambda *a, **k: _Resp(202))
|
|
25
|
+
monkeypatch.setattr(cli.requests, "get", lambda *a, **k: _Resp(200, {"ok": True, "count": 1}))
|
|
26
|
+
monkeypatch.setattr(cli.udp, "send_ping", lambda *a, **k: True)
|
|
27
|
+
assert cli.main(["doctor", "--api-key", "foxy_sk_test", "--backend", "http://x"]) == 0
|
|
28
|
+
out = capsys.readouterr().out
|
|
29
|
+
assert "Backend reachable" in out and "Ledger verified" in out
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_doctor_bad_key(monkeypatch):
|
|
33
|
+
monkeypatch.setattr(cli.requests, "post", lambda *a, **k: _Resp(401, text="Invalid API key"))
|
|
34
|
+
monkeypatch.setattr(cli.udp, "send_ping", lambda *a, **k: True)
|
|
35
|
+
assert cli.main(["doctor", "--api-key", "bad", "--backend", "http://x"]) == 1
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_doctor_no_key(monkeypatch):
|
|
39
|
+
monkeypatch.delenv("FOXY_API_KEY", raising=False)
|
|
40
|
+
assert cli.main(["doctor", "--backend", "http://x"]) == 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_doctor_backend_unreachable(monkeypatch):
|
|
44
|
+
def _boom(*a, **k):
|
|
45
|
+
raise requests.RequestException("no route to host")
|
|
46
|
+
monkeypatch.setattr(cli.requests, "post", _boom)
|
|
47
|
+
monkeypatch.setattr(cli.udp, "send_ping", lambda *a, **k: True)
|
|
48
|
+
assert cli.main(["doctor", "--api-key", "foxy_sk_test", "--backend", "http://x"]) == 1
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""End-to-end SDK integration test against a stdlib mock backend + fake fox.
|
|
2
|
+
|
|
3
|
+
Proves the full dispatch path without Postgres or Gemini:
|
|
4
|
+
• the instant `evaluating` UDP ping is emitted per audited call,
|
|
5
|
+
• the AsyncDispatcher POSTs a BATCH (a JSON list) to {endpoint}/v1/logs/batch
|
|
6
|
+
with a Bearer header, and each item carries only hashes / token_count /
|
|
7
|
+
policy_tag (never raw text),
|
|
8
|
+
• every audited call is eventually delivered (no drops across batches).
|
|
9
|
+
|
|
10
|
+
Grading is asynchronous in the real backend (the batch endpoint returns 202
|
|
11
|
+
{"status": "pending"} and the worker back-fills verdicts later), so the SDK gets
|
|
12
|
+
no verdict at ingest time — breach signalling is not part of this HTTP path.
|
|
13
|
+
|
|
14
|
+
Run: PYTHONPATH=sdk/src python sdk/tests/test_integration.py
|
|
15
|
+
or: pip install -e ./sdk && pytest sdk/tests/test_integration.py
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import socket
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
25
|
+
|
|
26
|
+
from foxy_audit import FoxyClient
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _MockBackend:
|
|
30
|
+
"""Captures POSTs and replies like the real batch endpoint (202, pending)."""
|
|
31
|
+
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self.requests: list[tuple[str, dict, list]] = [] # (path, headers, body)
|
|
34
|
+
handler = self._make_handler()
|
|
35
|
+
self.server = HTTPServer(("127.0.0.1", 0), handler)
|
|
36
|
+
self.port = self.server.server_address[1]
|
|
37
|
+
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
38
|
+
self.thread.start()
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def endpoint(self) -> str:
|
|
42
|
+
return f"http://127.0.0.1:{self.port}"
|
|
43
|
+
|
|
44
|
+
def _make_handler(self):
|
|
45
|
+
backend = self
|
|
46
|
+
|
|
47
|
+
class Handler(BaseHTTPRequestHandler):
|
|
48
|
+
def log_message(self, *_): # silence
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
def do_POST(self):
|
|
52
|
+
length = int(self.headers.get("Content-Length", 0))
|
|
53
|
+
body = json.loads(self.rfile.read(length) or b"[]")
|
|
54
|
+
backend.requests.append((self.path, dict(self.headers), body))
|
|
55
|
+
resp = json.dumps({"status": "pending", "count": len(body)}).encode()
|
|
56
|
+
self.send_response(202)
|
|
57
|
+
self.send_header("Content-Type", "application/json")
|
|
58
|
+
self.send_header("Content-Length", str(len(resp)))
|
|
59
|
+
self.end_headers()
|
|
60
|
+
self.wfile.write(resp)
|
|
61
|
+
|
|
62
|
+
return Handler
|
|
63
|
+
|
|
64
|
+
def stop(self):
|
|
65
|
+
self.server.shutdown()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _udp_listener():
|
|
69
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
70
|
+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
71
|
+
sock.bind(("127.0.0.1", 0))
|
|
72
|
+
sock.settimeout(0.5)
|
|
73
|
+
return sock, sock.getsockname()[1]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _recv_event(sock, timeout=5.0):
|
|
77
|
+
sock.settimeout(timeout)
|
|
78
|
+
try:
|
|
79
|
+
data, _ = sock.recvfrom(4096)
|
|
80
|
+
return json.loads(data.decode())
|
|
81
|
+
except socket.timeout:
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _delivered(backend) -> list:
|
|
86
|
+
"""All payload items received so far, across every batch POST."""
|
|
87
|
+
return [item for _, _, body in backend.requests for item in body]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _wait_for_delivered(backend, n, timeout=8.0):
|
|
91
|
+
deadline = time.time() + timeout
|
|
92
|
+
while len(_delivered(backend)) < n and time.time() < deadline:
|
|
93
|
+
time.sleep(0.02)
|
|
94
|
+
return len(_delivered(backend)) >= n
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_full_dispatch_batches_hashes_only():
|
|
98
|
+
backend = _MockBackend()
|
|
99
|
+
sock, udp_port = _udp_listener()
|
|
100
|
+
try:
|
|
101
|
+
foxy = FoxyClient(api_key="foxy_sk_test", endpoint=backend.endpoint, udp_port=udp_port)
|
|
102
|
+
|
|
103
|
+
@foxy.audit("hipaa_basic")
|
|
104
|
+
def ask(prompt: str) -> str:
|
|
105
|
+
return "a clinical summary response"
|
|
106
|
+
|
|
107
|
+
# ── first audited call ──
|
|
108
|
+
assert ask("a patient prompt") == "a clinical summary response"
|
|
109
|
+
|
|
110
|
+
pending = _recv_event(sock, 2.0)
|
|
111
|
+
assert pending and pending["event"] == "evaluating" and pending["policy"] == "hipaa_basic", pending
|
|
112
|
+
|
|
113
|
+
assert _wait_for_delivered(backend, 1), "backend never received the first batch"
|
|
114
|
+
path, headers, body = backend.requests[0]
|
|
115
|
+
assert path == "/v1/logs/batch", path
|
|
116
|
+
assert headers.get("Authorization") == "Bearer foxy_sk_test"
|
|
117
|
+
assert isinstance(body, list) and len(body) >= 1, body
|
|
118
|
+
|
|
119
|
+
item = body[0]
|
|
120
|
+
assert len(item["prompt_hash"]) == 64 and len(item["response_hash"]) == 64
|
|
121
|
+
assert item["policy_tag"] == "hipaa_basic" and isinstance(item["token_count"], int)
|
|
122
|
+
# zero-knowledge: no raw text leaves the host, under any key
|
|
123
|
+
wire = json.dumps(body)
|
|
124
|
+
assert "a patient prompt" not in wire and "clinical summary" not in wire
|
|
125
|
+
|
|
126
|
+
# ── second call: also delivered (a later batch or coalesced) ──
|
|
127
|
+
ask("another prompt")
|
|
128
|
+
pending2 = _recv_event(sock, 2.0)
|
|
129
|
+
assert pending2 and pending2["event"] == "evaluating", pending2
|
|
130
|
+
|
|
131
|
+
assert _wait_for_delivered(backend, 2), "second call was never delivered"
|
|
132
|
+
delivered = _delivered(backend)
|
|
133
|
+
assert len(delivered) == 2
|
|
134
|
+
assert all(len(d["prompt_hash"]) == 64 for d in delivered)
|
|
135
|
+
# every batch went to the batch endpoint with the same Bearer key
|
|
136
|
+
assert all(p == "/v1/logs/batch" for p, _, _ in backend.requests)
|
|
137
|
+
assert all(h.get("Authorization") == "Bearer foxy_sk_test"
|
|
138
|
+
for _, h, _ in backend.requests)
|
|
139
|
+
print("integration: batched hash-only dispatch PASS")
|
|
140
|
+
finally:
|
|
141
|
+
sock.close()
|
|
142
|
+
backend.stop()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
if __name__ == "__main__":
|
|
146
|
+
test_full_dispatch_batches_hashes_only()
|
|
147
|
+
print("\nINTEGRATION TEST PASSED")
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""PII signal detection — the expanded regex layer (Phase 5 · 5J).
|
|
2
|
+
|
|
3
|
+
Presidio is an optional extra; these cover the always-on, dependency-free layer.
|
|
4
|
+
Only SIGNAL labels are asserted — raw values never leave the client.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from foxy_audit.pii import detect_pii
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_email_and_ssn():
|
|
13
|
+
assert "email" in detect_pii("reach me at jane.doe@acme.co", "")
|
|
14
|
+
assert "ssn_pattern" in detect_pii("", "ssn 123-45-6789")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_phone():
|
|
18
|
+
assert "phone" in detect_pii("call (415) 555-2671 today", "")
|
|
19
|
+
assert "phone" in detect_pii("", "+1 415-555-2671")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_credit_card_luhn_gated():
|
|
23
|
+
assert "credit_card" in detect_pii("card 4242 4242 4242 4242", "") # valid Luhn
|
|
24
|
+
assert "credit_card" not in detect_pii("ref 4242 4242 4242 4241", "") # fails Luhn
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_ip_address():
|
|
28
|
+
assert "ip_address" in detect_pii("connected from 192.168.1.100", "")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_clean_text_has_no_signals():
|
|
32
|
+
assert detect_pii("what is the capital of France?", "Paris.") == []
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_signals_are_deduped_and_sorted():
|
|
36
|
+
sig = detect_pii("a@b.com and c@d.com", "")
|
|
37
|
+
assert sig == sorted(set(sig)) and sig.count("email") == 1
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Unit tests for the Foxy Audit SDK that need no backend.
|
|
2
|
+
|
|
3
|
+
Run with: pip install -e ./sdk[test] && pytest sdk/tests
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import socket
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
from foxy_audit import FoxyClient
|
|
11
|
+
from foxy_audit import hashing
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _udp_listener():
|
|
15
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
16
|
+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
17
|
+
sock.bind(("127.0.0.1", 0)) # ephemeral port
|
|
18
|
+
sock.settimeout(2.0)
|
|
19
|
+
return sock, sock.getsockname()[1]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_hashing_is_64_hex():
|
|
23
|
+
h = hashing.sha256_hex("hello")
|
|
24
|
+
assert len(h) == 64 and all(c in "0123456789abcdef" for c in h)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_return_value_passes_through():
|
|
28
|
+
foxy = FoxyClient(api_key="", desktop_ping=False)
|
|
29
|
+
|
|
30
|
+
@foxy.audit("demo")
|
|
31
|
+
def f(prompt: str) -> str:
|
|
32
|
+
return prompt.upper()
|
|
33
|
+
|
|
34
|
+
assert f("hi") == "HI"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_host_exception_not_masked():
|
|
38
|
+
foxy = FoxyClient(api_key="", desktop_ping=False)
|
|
39
|
+
|
|
40
|
+
@foxy.audit("demo")
|
|
41
|
+
def boom(prompt: str) -> str:
|
|
42
|
+
raise ValueError("host error")
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
boom("x")
|
|
46
|
+
assert False, "expected ValueError"
|
|
47
|
+
except ValueError as e:
|
|
48
|
+
assert str(e) == "host error"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_evaluating_udp_emitted():
|
|
52
|
+
sock, port = _udp_listener()
|
|
53
|
+
try:
|
|
54
|
+
foxy = FoxyClient(api_key="", udp_port=port) # no key → HTTP disabled, UDP still fires
|
|
55
|
+
|
|
56
|
+
@foxy.audit("hipaa_basic")
|
|
57
|
+
def ask(prompt: str) -> str:
|
|
58
|
+
return "a response"
|
|
59
|
+
|
|
60
|
+
ask("a prompt")
|
|
61
|
+
data, _ = sock.recvfrom(4096)
|
|
62
|
+
msg = json.loads(data.decode())
|
|
63
|
+
# the instant ping is "evaluating" (grading pending) — never a premature verdict
|
|
64
|
+
assert msg["event"] == "evaluating"
|
|
65
|
+
assert msg["policy"] == "hipaa_basic"
|
|
66
|
+
assert isinstance(msg["tokens"], int)
|
|
67
|
+
assert "ts" in msg
|
|
68
|
+
finally:
|
|
69
|
+
sock.close()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_agent_included_in_payload(monkeypatch):
|
|
73
|
+
"""audit(policy, agent=…) puts the agent into the backend payload (6B)."""
|
|
74
|
+
from foxy_audit import dispatch
|
|
75
|
+
captured = {}
|
|
76
|
+
monkeypatch.setattr(dispatch, "submit", lambda cfg, payload: captured.update(payload))
|
|
77
|
+
foxy = FoxyClient(api_key="foxy_sk_test", desktop_ping=False) # enabled → dispatch fires
|
|
78
|
+
|
|
79
|
+
@foxy.audit("demo", agent="gpt-4o")
|
|
80
|
+
def ask(prompt: str) -> str:
|
|
81
|
+
return "resp"
|
|
82
|
+
|
|
83
|
+
ask("hi")
|
|
84
|
+
assert captured.get("agent") == "gpt-4o"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_payload_omits_vestigial_fields(monkeypatch):
|
|
88
|
+
"""The backend ignores chain_hash/timestamp; the SDK must not send them (7A-1)."""
|
|
89
|
+
from foxy_audit import dispatch
|
|
90
|
+
captured = {}
|
|
91
|
+
monkeypatch.setattr(dispatch, "submit", lambda cfg, payload: captured.update(payload))
|
|
92
|
+
foxy = FoxyClient(api_key="foxy_sk_test", desktop_ping=False) # enabled → dispatch fires
|
|
93
|
+
|
|
94
|
+
@foxy.audit("demo")
|
|
95
|
+
def ask(prompt: str) -> str:
|
|
96
|
+
return "resp"
|
|
97
|
+
|
|
98
|
+
ask("hi")
|
|
99
|
+
assert captured, "expected a payload to be dispatched"
|
|
100
|
+
assert "chain_hash" not in captured
|
|
101
|
+
assert "timestamp" not in captured
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_agent_absent_by_default(monkeypatch):
|
|
105
|
+
from foxy_audit import dispatch
|
|
106
|
+
captured = {}
|
|
107
|
+
monkeypatch.setattr(dispatch, "submit", lambda cfg, payload: captured.update(payload))
|
|
108
|
+
foxy = FoxyClient(api_key="foxy_sk_test", desktop_ping=False)
|
|
109
|
+
|
|
110
|
+
@foxy.audit("demo")
|
|
111
|
+
def ask(prompt: str) -> str:
|
|
112
|
+
return "resp"
|
|
113
|
+
|
|
114
|
+
ask("hi")
|
|
115
|
+
assert not captured.get("agent") # no agent when not specified
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_no_key_means_http_disabled_but_udp_enabled():
|
|
119
|
+
foxy = FoxyClient(api_key="")
|
|
120
|
+
assert foxy.enabled is False
|
|
121
|
+
foxy2 = FoxyClient(api_key="foxy_sk_test")
|
|
122
|
+
assert foxy2.enabled is True
|