navigator-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- navigator_cli-0.1.0/PKG-INFO +15 -0
- navigator_cli-0.1.0/README.md +141 -0
- navigator_cli-0.1.0/app/__init__.py +1 -0
- navigator_cli-0.1.0/app/catalogue_client.py +127 -0
- navigator_cli-0.1.0/app/cli.py +376 -0
- navigator_cli-0.1.0/app/cli_auth.py +93 -0
- navigator_cli-0.1.0/app/keys.py +98 -0
- navigator_cli-0.1.0/app/main.py +451 -0
- navigator_cli-0.1.0/app/member_auth.py +217 -0
- navigator_cli-0.1.0/app/nl_query.py +122 -0
- navigator_cli-0.1.0/app/runner.py +56 -0
- navigator_cli-0.1.0/app/skills_registry.py +105 -0
- navigator_cli-0.1.0/app/static/styles.css +1054 -0
- navigator_cli-0.1.0/app/submissions.py +97 -0
- navigator_cli-0.1.0/app/templates/index.html +492 -0
- navigator_cli-0.1.0/app/templates/login.html +43 -0
- navigator_cli-0.1.0/app/templates/source.html +62 -0
- navigator_cli-0.1.0/app/tools_client.py +60 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/PKG-INFO +15 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/SOURCES.txt +32 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/dependency_links.txt +1 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/entry_points.txt +2 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/requires.txt +11 -0
- navigator_cli-0.1.0/navigator_cli.egg-info/top_level.txt +1 -0
- navigator_cli-0.1.0/pyproject.toml +43 -0
- navigator_cli-0.1.0/setup.cfg +4 -0
- navigator_cli-0.1.0/tests/test_app.py +173 -0
- navigator_cli-0.1.0/tests/test_catalogue_client.py +189 -0
- navigator_cli-0.1.0/tests/test_cli.py +193 -0
- navigator_cli-0.1.0/tests/test_data_prefix.py +63 -0
- navigator_cli-0.1.0/tests/test_member_auth.py +243 -0
- navigator_cli-0.1.0/tests/test_runner.py +61 -0
- navigator_cli-0.1.0/tests/test_skills_registry.py +79 -0
- navigator_cli-0.1.0/tests/test_submissions.py +94 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: navigator-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Navigator CLI + Data Navigator service: query public data sources from your agent
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: fastapi<1,>=0.115
|
|
7
|
+
Requires-Dist: httpx<1,>=0.27
|
|
8
|
+
Requires-Dist: jinja2<4,>=3.1
|
|
9
|
+
Requires-Dist: keyring<26,>=25
|
|
10
|
+
Requires-Dist: markdown>=3.10.2
|
|
11
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
12
|
+
Requires-Dist: pyyaml<7,>=6
|
|
13
|
+
Requires-Dist: uvicorn[standard]<1,>=0.30
|
|
14
|
+
Provides-Extra: test
|
|
15
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# Data Navigator
|
|
2
|
+
|
|
3
|
+
Members-only catalogue of skill files that teach AI agents to query public
|
|
4
|
+
databases, plus the executor that runs the queries. Three faces — web UI, the
|
|
5
|
+
`navigator` CLI, and (Phase 3) MCP — all over one HTTP API. See `SPEC.md` for
|
|
6
|
+
the full design; this README covers running and validating the local app.
|
|
7
|
+
|
|
8
|
+
## Layout
|
|
9
|
+
|
|
10
|
+
The skill tree is the single source of truth:
|
|
11
|
+
|
|
12
|
+
```text
|
|
13
|
+
skills/<jurisdiction>/<source-slug>/
|
|
14
|
+
├── SKILL.md # agent-facing playbook: inputs, output shape, gotchas
|
|
15
|
+
├── meta.yaml # validated catalogue record (app/skills_registry.py)
|
|
16
|
+
└── adapter.py # run(input, ctx) — executes the query
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
- `skills/_template/` — copy this to add a source; every field is documented.
|
|
20
|
+
- `skills/_meta/navigator-data/` — the meta-skill members drop into
|
|
21
|
+
`~/.claude/skills/` so their agent knows the find → show → query loop.
|
|
22
|
+
- `app/skills_registry.py` — walks the tree, validates every `meta.yaml`
|
|
23
|
+
(pydantic), builds the registry. Invalid skill dirs fail the app at startup.
|
|
24
|
+
- `app/runner.py` — dynamic adapter loading + `AdapterContext.get_key()`
|
|
25
|
+
(OS keychain / `NAVIGATOR_KEY_<NAME>` env; keys never enter agent context).
|
|
26
|
+
|
|
27
|
+
Source runtimes: `local` (public or BYO-key, runs on the member's machine,
|
|
28
|
+
unmetered) and `hosted` (our key, runs server-side, metered — e.g.
|
|
29
|
+
OpenSanctions).
|
|
30
|
+
|
|
31
|
+
## Run locally
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
uv sync --extra test
|
|
35
|
+
uv run uvicorn app.main:app --reload --port 8000
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Open `http://localhost:8000/data`. Set `OPENROUTER_API_KEY` to enable the
|
|
39
|
+
natural-language query bar (`/api/query/natural` returns 503 without it —
|
|
40
|
+
there is no fallback planner).
|
|
41
|
+
|
|
42
|
+
## Run with Docker
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
docker compose up --build
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## CLI
|
|
49
|
+
|
|
50
|
+
One unified `navigator` CLI for the whole product: `navigator data …` is Data
|
|
51
|
+
Navigator, `navigator tools …` (Phase 3) is osint-navigator.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
navigator auth login # magic link → PAT in the OS keychain
|
|
55
|
+
navigator auth status # tier + quota state
|
|
56
|
+
navigator data find company # search the catalogue
|
|
57
|
+
navigator data show no/brreg/enheter # print the source's SKILL.md + meta
|
|
58
|
+
navigator query no/brreg/enheter --input '{"navn":"Equinor","size":3}'
|
|
59
|
+
navigator query no/brreg/enheter --input '{"navn":"Equinor"}' --out companies.csv
|
|
60
|
+
navigator tools find "satellite imagery" # OSINT tool search (osint-navigator API)
|
|
61
|
+
navigator tools show <tool-id> # full tool record + documentation
|
|
62
|
+
navigator keys missing # BYO keys you still need to set
|
|
63
|
+
navigator keys set companies-house # store a key in the OS keychain
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`--out <file>.json|.csv` writes the records to the file and prints a compact
|
|
67
|
+
summary instead of the full payload. Hosted sources route to the Data
|
|
68
|
+
Navigator server (`DATANAV_BASE_URL`) with the stored PAT.
|
|
69
|
+
|
|
70
|
+
There is no install step: with `DATANAV_CATALOGUE=remote` the CLI fetches
|
|
71
|
+
skill bundles from the authed API and caches them under `~/.navigator/cache/`
|
|
72
|
+
(re-fetched when a source's meta changes). Default is `local` (this checkout)
|
|
73
|
+
for dev and smoke tests. The only thing a member installs is the meta-skill.
|
|
74
|
+
|
|
75
|
+
## Membership (Phase 2)
|
|
76
|
+
|
|
77
|
+
Everything is member-gated in production (`DATANAV_AUTH=on`; default off for
|
|
78
|
+
local dev). Auth is owned by osint-navigator — this service validates Bearer
|
|
79
|
+
PATs / the shared `osint_session` cookie via its internal introspection API
|
|
80
|
+
and requires the `data` tier (above osint pro). Metering: UI NL queries +
|
|
81
|
+
try-it burn `data_ui`; hosted-source queries burn `data_hosted`; public + BYO
|
|
82
|
+
local execution is never metered. Quota storage lives in osint-navigator.
|
|
83
|
+
|
|
84
|
+
Cross-service proof: `uv run python scripts/integration_check_membership.py`
|
|
85
|
+
(boots osint-navigator's internal API from the sibling checkout and verifies
|
|
86
|
+
401/403/tier/quota behavior end-to-end).
|
|
87
|
+
|
|
88
|
+
## MCP, meta-skill, rollout
|
|
89
|
+
|
|
90
|
+
- **MCP**: `data_find_source` / `data_show_source` / `data_query` live on
|
|
91
|
+
osint-navigator's remote MCP and proxy here over the service channel. Local
|
|
92
|
+
sources are unmetered over MCP (the member's agent reasons; upstream is free).
|
|
93
|
+
- **Meta-skill**: members download it from the UI ("Agent setup") or
|
|
94
|
+
`GET /api/meta-skill` and drop it into `~/.claude/skills/navigator-data/`.
|
|
95
|
+
- **Rollout**: the Data toggle in osint-navigator's header is admin-only
|
|
96
|
+
(`ADMIN_EMAILS` web sessions) until launch; `/data` is reverse-proxied from
|
|
97
|
+
osint-navigator when `DATA_NAVIGATOR_URL` is set.
|
|
98
|
+
- **Deploy**: operator steps in `docs/deploy/phase2-deploy-runbook.html`.
|
|
99
|
+
|
|
100
|
+
## API
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
curl -s "http://localhost:8000/api/registry"
|
|
104
|
+
curl -s "http://localhost:8000/api/sources/no/brreg/enheter" # meta + skill_md
|
|
105
|
+
curl -s -X POST "http://localhost:8000/api/query/no/brreg/enheter" \
|
|
106
|
+
-H "Content-Type: application/json" -d '{"navn":"Equinor","size":3}'
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Validation
|
|
110
|
+
|
|
111
|
+
First, validate the skill tree (both smoke scripts assume this passes):
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
uv run python scripts/validate_skills.py
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The validation gate is three live public APIs returning records:
|
|
118
|
+
|
|
119
|
+
- `no/brreg/enheter` — Norway Brønnøysund companies
|
|
120
|
+
- `us/usaspending/awards` — USAspending federal awards
|
|
121
|
+
- `global/gleif/lei-records` — GLEIF LEI records
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
uv run pytest
|
|
125
|
+
DATANAV_BASE_URL=http://localhost:8000 uv run python scripts/smoke_public_apis.py
|
|
126
|
+
NAVIGATOR_CMD="uv run navigator" uv run python scripts/smoke_cli_public_apis.py
|
|
127
|
+
bash scripts/smoke_docker_image.sh
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Deployment
|
|
131
|
+
|
|
132
|
+
`render.yaml` defines a Docker web service (health check `/health`, port from
|
|
133
|
+
`$PORT`). Server secrets: `OPENROUTER_API_KEY` (NL planner) and
|
|
134
|
+
`NAVIGATOR_KEY_OPENSANCTIONS` (hosted OpenSanctions sources).
|
|
135
|
+
|
|
136
|
+
Phase 2 (membership wiring: auth introspection against osint-navigator,
|
|
137
|
+
metered quotas, submissions form) and Phase 3 (MCP, catalogue growth) are
|
|
138
|
+
specified in `SPEC.md` §9.
|
|
139
|
+
|
|
140
|
+
Spotlight integration is intentionally not implemented here — separate
|
|
141
|
+
partner-reviewed PR after the OpenSanctions output contract is stable.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Data Navigator local app."""
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Remote catalogue for the CLI: fetch-on-demand skills, cached locally.
|
|
2
|
+
|
|
3
|
+
There is no install step. `find`/`show`/`query` pull the registry and skill
|
|
4
|
+
bundles from the authed Data Navigator API and cache them under
|
|
5
|
+
~/.navigator/cache/. A cached bundle is re-fetched when the registry entry's
|
|
6
|
+
`last_tested` or meta content changes; offline, the stale cache is used with
|
|
7
|
+
a warning on stderr.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
from app.cli_auth import get_pat
|
|
20
|
+
from app.skills_registry import Source
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def cache_dir() -> Path:
|
|
24
|
+
return Path(os.getenv("NAVIGATOR_CACHE_DIR", str(Path.home() / ".navigator" / "cache")))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _base_url() -> str:
|
|
28
|
+
return os.getenv("DATANAV_BASE_URL", "https://navigator.indicator.media/data").rstrip("/")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _headers() -> dict:
|
|
32
|
+
pat = get_pat()
|
|
33
|
+
if not pat:
|
|
34
|
+
raise RuntimeError("Not logged in. Run: navigator auth login")
|
|
35
|
+
return {"Authorization": f"Bearer {pat}"}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _get(path: str) -> dict:
|
|
39
|
+
response = httpx.get(f"{_base_url()}{path}", headers=_headers(), timeout=30)
|
|
40
|
+
if response.status_code == 401:
|
|
41
|
+
raise RuntimeError("Session expired or invalid. Run: navigator auth login")
|
|
42
|
+
if response.status_code == 403:
|
|
43
|
+
detail = response.json().get("detail", {})
|
|
44
|
+
url = detail.get("upgrade_url", "") if isinstance(detail, dict) else ""
|
|
45
|
+
raise RuntimeError(f"Data Navigator membership required. {url}".strip())
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
return response.json()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _meta_fingerprint(meta: dict) -> str:
|
|
51
|
+
return hashlib.sha256(
|
|
52
|
+
json.dumps(meta, sort_keys=True, ensure_ascii=False).encode()
|
|
53
|
+
).hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def fetch_registry() -> dict:
|
|
57
|
+
"""Fetch the registry, falling back to the cached copy when offline."""
|
|
58
|
+
registry_path = cache_dir() / "registry.json"
|
|
59
|
+
try:
|
|
60
|
+
registry = _get("/api/registry")
|
|
61
|
+
except httpx.HTTPError as exc:
|
|
62
|
+
if registry_path.exists():
|
|
63
|
+
print(f"navigator: offline ({exc}); using cached catalogue", file=sys.stderr)
|
|
64
|
+
return json.loads(registry_path.read_text(encoding="utf-8"))
|
|
65
|
+
raise RuntimeError(f"Could not reach Data Navigator and no cache exists: {exc}") from exc
|
|
66
|
+
registry_path.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
registry_path.write_text(json.dumps(registry, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
68
|
+
return registry
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _bundle_dir(source_id: str) -> Path:
|
|
72
|
+
return cache_dir() / source_id.replace("/", "--")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fetch_bundle(source_id: str, meta: dict) -> Path:
|
|
76
|
+
"""Ensure the skill bundle is cached and current; return its directory."""
|
|
77
|
+
target = _bundle_dir(source_id)
|
|
78
|
+
fingerprint_path = target / ".fingerprint"
|
|
79
|
+
fingerprint = _meta_fingerprint(meta)
|
|
80
|
+
|
|
81
|
+
if fingerprint_path.exists() and fingerprint_path.read_text() == fingerprint:
|
|
82
|
+
return target
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
bundle = _get(f"/api/sources/{source_id}/bundle")
|
|
86
|
+
except httpx.HTTPError as exc:
|
|
87
|
+
if (target / "meta.json").exists():
|
|
88
|
+
print(f"navigator: offline ({exc}); using cached skill", file=sys.stderr)
|
|
89
|
+
return target
|
|
90
|
+
raise RuntimeError(f"Could not fetch skill bundle for {source_id}: {exc}") from exc
|
|
91
|
+
|
|
92
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
93
|
+
(target / "meta.json").write_text(
|
|
94
|
+
json.dumps(bundle["meta"], ensure_ascii=False, indent=2), encoding="utf-8"
|
|
95
|
+
)
|
|
96
|
+
(target / "SKILL.md").write_text(bundle["skill_md"], encoding="utf-8")
|
|
97
|
+
adapter = bundle.get("adapter_py")
|
|
98
|
+
adapter_path = target / "adapter.py"
|
|
99
|
+
if adapter:
|
|
100
|
+
adapter_path.write_text(adapter, encoding="utf-8")
|
|
101
|
+
elif adapter_path.exists():
|
|
102
|
+
adapter_path.unlink()
|
|
103
|
+
fingerprint_path.write_text(fingerprint)
|
|
104
|
+
return target
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def remote_catalogue() -> list[Source]:
|
|
108
|
+
"""Registry entries as Source models. `dir` points at nothing yet —
|
|
109
|
+
it is bound to the cached bundle dir on demand by remote_source()."""
|
|
110
|
+
registry = fetch_registry()
|
|
111
|
+
sources = []
|
|
112
|
+
for entry in registry.get("sources", []):
|
|
113
|
+
data = {k: v for k, v in entry.items() if k != "queryable"}
|
|
114
|
+
sources.append(Source(**data))
|
|
115
|
+
return sources
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def remote_source(source_id: str) -> Source:
|
|
119
|
+
"""Resolve one source with its cached bundle dir bound (for local execution)."""
|
|
120
|
+
registry = fetch_registry()
|
|
121
|
+
entry = next((s for s in registry.get("sources", []) if s["id"] == source_id), None)
|
|
122
|
+
if entry is None:
|
|
123
|
+
raise ValueError(f"Unknown source: {source_id}")
|
|
124
|
+
meta = {k: v for k, v in entry.items() if k != "queryable"}
|
|
125
|
+
source = Source(**meta)
|
|
126
|
+
source.dir = fetch_bundle(source_id, meta)
|
|
127
|
+
return source
|
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
"""Agent-facing `navigator` CLI for Data Navigator."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import csv
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
from getpass import getpass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Sequence
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from app import cli_auth
|
|
17
|
+
from app.keys import KeyStoreError, key_state, navigator_env_var, set_key, unset_key
|
|
18
|
+
from app.runner import MissingKeyError, execute
|
|
19
|
+
from app.skills_registry import Source, load_sources, source_payload
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _remote_mode() -> bool:
|
|
23
|
+
"""remote = authed API + ~/.navigator/cache (members); local = repo checkout (dev)."""
|
|
24
|
+
return os.getenv("DATANAV_CATALOGUE", "local") == "remote"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _catalogue() -> list[Source]:
|
|
28
|
+
if _remote_mode():
|
|
29
|
+
from app.catalogue_client import remote_catalogue
|
|
30
|
+
|
|
31
|
+
return remote_catalogue()
|
|
32
|
+
return load_sources()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _get_source(source_id: str) -> Source:
|
|
36
|
+
if _remote_mode():
|
|
37
|
+
from app.catalogue_client import remote_source
|
|
38
|
+
|
|
39
|
+
return remote_source(source_id)
|
|
40
|
+
source = next((s for s in load_sources() if s.id == source_id), None)
|
|
41
|
+
if not source:
|
|
42
|
+
raise ValueError(f"Unknown source: {source_id}")
|
|
43
|
+
return source
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _parse_json_input(raw: str) -> dict[str, Any]:
|
|
47
|
+
try:
|
|
48
|
+
value = json.loads(raw)
|
|
49
|
+
except json.JSONDecodeError as exc:
|
|
50
|
+
raise ValueError(f"--input must be valid JSON: {exc.msg}") from exc
|
|
51
|
+
if not isinstance(value, dict):
|
|
52
|
+
raise ValueError("--input must be a JSON object")
|
|
53
|
+
return value
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _print_json(payload: Any) -> None:
|
|
57
|
+
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=False))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _print_sources(sources: list[Source], as_json: bool) -> None:
|
|
61
|
+
if as_json:
|
|
62
|
+
_print_json({"sources": [source_payload(s) for s in sources], "count": len(sources)})
|
|
63
|
+
return
|
|
64
|
+
for source in sources:
|
|
65
|
+
marker = "queryable" if source.queryable else source.status
|
|
66
|
+
print(f"{source.id}\t{marker}\t{source.title}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _run_hosted_query(source: Source, payload: dict[str, Any]) -> dict[str, Any]:
|
|
70
|
+
base = os.getenv("DATANAV_BASE_URL", "http://localhost:8000").rstrip("/")
|
|
71
|
+
headers = {}
|
|
72
|
+
pat = cli_auth.get_pat()
|
|
73
|
+
if pat:
|
|
74
|
+
headers["Authorization"] = f"Bearer {pat}"
|
|
75
|
+
try:
|
|
76
|
+
response = httpx.post(
|
|
77
|
+
f"{base}/api/query/{source.id}", json=payload, headers=headers, timeout=60
|
|
78
|
+
)
|
|
79
|
+
except httpx.HTTPError as exc:
|
|
80
|
+
raise RuntimeError(f"Could not reach Data Navigator server at {base}: {exc}") from exc
|
|
81
|
+
if response.status_code == 401:
|
|
82
|
+
raise RuntimeError("Membership required for hosted sources. Run: navigator auth login")
|
|
83
|
+
if response.status_code >= 400:
|
|
84
|
+
try:
|
|
85
|
+
detail = response.json().get("detail")
|
|
86
|
+
except (json.JSONDecodeError, ValueError):
|
|
87
|
+
detail = response.text[:500]
|
|
88
|
+
raise RuntimeError(f"Server returned HTTP {response.status_code}: {detail}")
|
|
89
|
+
return response.json()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _run_query(source_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
93
|
+
source = _get_source(source_id)
|
|
94
|
+
if source.runtime == "hosted":
|
|
95
|
+
return _run_hosted_query(source, payload)
|
|
96
|
+
try:
|
|
97
|
+
return execute(source, payload)
|
|
98
|
+
except MissingKeyError as exc:
|
|
99
|
+
raise ValueError(str(exc)) from exc
|
|
100
|
+
except httpx.HTTPStatusError as exc:
|
|
101
|
+
raise RuntimeError(
|
|
102
|
+
f"Upstream returned HTTP {exc.response.status_code}: {exc.response.text[:500]}"
|
|
103
|
+
) from exc
|
|
104
|
+
except httpx.HTTPError as exc:
|
|
105
|
+
raise RuntimeError(f"Upstream request failed: {exc}") from exc
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _write_out(result: dict, out: str) -> dict:
|
|
109
|
+
path = Path(out)
|
|
110
|
+
records = result.get("records", [])
|
|
111
|
+
if path.suffix == ".csv":
|
|
112
|
+
keys = sorted({k for r in records for k in r})
|
|
113
|
+
with path.open("w", newline="", encoding="utf-8") as fh:
|
|
114
|
+
writer = csv.DictWriter(fh, fieldnames=keys)
|
|
115
|
+
writer.writeheader()
|
|
116
|
+
for r in records:
|
|
117
|
+
writer.writerow(
|
|
118
|
+
{k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in r.items()}
|
|
119
|
+
)
|
|
120
|
+
else:
|
|
121
|
+
path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
122
|
+
return {
|
|
123
|
+
"source_id": result.get("source_id"),
|
|
124
|
+
"records": len(records),
|
|
125
|
+
"fields": sorted({k for r in records[:1] for k in r}),
|
|
126
|
+
"out": str(path),
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _cmd_data_find(args: argparse.Namespace) -> int:
|
|
131
|
+
needle = " ".join(args.query).lower()
|
|
132
|
+
sources = _catalogue()
|
|
133
|
+
if needle:
|
|
134
|
+
sources = [
|
|
135
|
+
s
|
|
136
|
+
for s in sources
|
|
137
|
+
if needle
|
|
138
|
+
in " ".join([s.id, s.title, s.description, s.category, s.jurisdiction, s.notes]).lower()
|
|
139
|
+
]
|
|
140
|
+
_print_sources(sources, args.json)
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _cmd_data_show(args: argparse.Namespace) -> int:
|
|
145
|
+
source = _get_source(args.source_id)
|
|
146
|
+
skill_md = (source.dir / "SKILL.md").read_text(encoding="utf-8")
|
|
147
|
+
print(skill_md)
|
|
148
|
+
print("---")
|
|
149
|
+
_print_json(source_payload(source))
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _cmd_query(args: argparse.Namespace) -> int:
|
|
154
|
+
payload = _parse_json_input(args.input)
|
|
155
|
+
result = _run_query(args.source_id, payload)
|
|
156
|
+
if getattr(args, "out", None):
|
|
157
|
+
_print_json(_write_out(result, args.out))
|
|
158
|
+
else:
|
|
159
|
+
_print_json(result)
|
|
160
|
+
return 0
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _cmd_tools_find(args: argparse.Namespace) -> int:
|
|
164
|
+
from app.tools_client import find_tools
|
|
165
|
+
|
|
166
|
+
result = find_tools(" ".join(args.query), category=args.category, limit=args.limit)
|
|
167
|
+
if args.json:
|
|
168
|
+
_print_json(result)
|
|
169
|
+
return 0
|
|
170
|
+
for tool in result.get("tools", []):
|
|
171
|
+
print(f"{tool.get('tool_id')}\t{tool.get('tool_name')}\t{tool.get('tool_url')}")
|
|
172
|
+
return 0
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _cmd_tools_show(args: argparse.Namespace) -> int:
|
|
176
|
+
from app.tools_client import show_tool
|
|
177
|
+
|
|
178
|
+
tool = show_tool(args.tool_id)
|
|
179
|
+
if args.json:
|
|
180
|
+
_print_json(tool)
|
|
181
|
+
return 0
|
|
182
|
+
print(f"# {tool.get('tool_name') or args.tool_id}")
|
|
183
|
+
for field in ("tool_url", "category", "pricing", "status"):
|
|
184
|
+
if tool.get(field):
|
|
185
|
+
print(f"{field}: {tool[field]}")
|
|
186
|
+
if tool.get("tags"):
|
|
187
|
+
print(f"tags: {', '.join(tool['tags'])}")
|
|
188
|
+
if tool.get("short_description"):
|
|
189
|
+
print(f"\n{tool['short_description']}")
|
|
190
|
+
if tool.get("documentation"):
|
|
191
|
+
print(f"\n{tool['documentation']}")
|
|
192
|
+
return 0
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _cmd_auth_login(args: argparse.Namespace) -> int:
|
|
196
|
+
email = args.email or input("Email on your membership: ").strip()
|
|
197
|
+
if not email:
|
|
198
|
+
raise ValueError("email is required")
|
|
199
|
+
result = cli_auth.login(email)
|
|
200
|
+
_print_json(result)
|
|
201
|
+
return 0
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _cmd_auth_status(args: argparse.Namespace) -> int:
|
|
205
|
+
_print_json(cli_auth.status())
|
|
206
|
+
return 0
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _cmd_auth_logout(args: argparse.Namespace) -> int:
|
|
210
|
+
_print_json(cli_auth.logout())
|
|
211
|
+
return 0
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _cmd_keys_missing(args: argparse.Namespace) -> int:
|
|
215
|
+
missing = []
|
|
216
|
+
for source in _catalogue():
|
|
217
|
+
if source.auth.type == "api_key" and source.auth.key_name:
|
|
218
|
+
state = key_state(source.auth.key_name)
|
|
219
|
+
if state.configured:
|
|
220
|
+
continue
|
|
221
|
+
missing.append(
|
|
222
|
+
{
|
|
223
|
+
"source_id": source.id,
|
|
224
|
+
"key_name": source.auth.key_name,
|
|
225
|
+
"env_var": navigator_env_var(source.auth.key_name),
|
|
226
|
+
"setup_url": source.auth.setup_url,
|
|
227
|
+
"setup_help": source.auth.setup_help,
|
|
228
|
+
}
|
|
229
|
+
)
|
|
230
|
+
_print_json({"missing": missing, "count": len(missing)})
|
|
231
|
+
return 0
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _configured_key_names() -> list[str]:
|
|
235
|
+
names = sorted(
|
|
236
|
+
{
|
|
237
|
+
source.auth.key_name
|
|
238
|
+
for source in _catalogue()
|
|
239
|
+
if source.auth.type == "api_key" and source.auth.key_name
|
|
240
|
+
}
|
|
241
|
+
)
|
|
242
|
+
return [name for name in names if name]
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _cmd_keys_list(args: argparse.Namespace) -> int:
|
|
246
|
+
states = [key_state(name).__dict__ for name in _configured_key_names()]
|
|
247
|
+
_print_json({"keys": states, "count": len(states)})
|
|
248
|
+
return 0
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _cmd_keys_set(args: argparse.Namespace) -> int:
|
|
252
|
+
value = sys.stdin.read().strip() if args.stdin else None
|
|
253
|
+
if value is None:
|
|
254
|
+
value = getpass(f"{args.name}: ")
|
|
255
|
+
set_key(args.name, value)
|
|
256
|
+
_print_json({"name": args.name, "stored": True, "source": "keyring"})
|
|
257
|
+
return 0
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _cmd_keys_get(args: argparse.Namespace) -> int:
|
|
261
|
+
if not args.reveal:
|
|
262
|
+
raise ValueError("Refusing to reveal a key without --reveal")
|
|
263
|
+
from app.keys import get_key
|
|
264
|
+
|
|
265
|
+
value = get_key(args.name)
|
|
266
|
+
if not value:
|
|
267
|
+
raise ValueError(f"Key is not configured: {args.name}")
|
|
268
|
+
print(value)
|
|
269
|
+
return 0
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _cmd_keys_unset(args: argparse.Namespace) -> int:
|
|
273
|
+
unset_key(args.name)
|
|
274
|
+
_print_json({"name": args.name, "deleted": True})
|
|
275
|
+
return 0
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _add_query_arguments(parser: argparse.ArgumentParser) -> None:
|
|
279
|
+
parser.add_argument("source_id")
|
|
280
|
+
parser.add_argument("--input", required=True, help="JSON object input for the source adapter")
|
|
281
|
+
parser.add_argument(
|
|
282
|
+
"--out",
|
|
283
|
+
help="Write records to this file (.json or .csv) and print a compact summary",
|
|
284
|
+
)
|
|
285
|
+
parser.set_defaults(func=_cmd_query)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
289
|
+
parser = argparse.ArgumentParser(
|
|
290
|
+
prog="navigator",
|
|
291
|
+
description="Navigator CLI — Data mode.",
|
|
292
|
+
)
|
|
293
|
+
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
294
|
+
|
|
295
|
+
data = subcommands.add_parser("data", help="Data Navigator commands")
|
|
296
|
+
data_subcommands = data.add_subparsers(dest="data_command", required=True)
|
|
297
|
+
|
|
298
|
+
data_find = data_subcommands.add_parser("find", help="Search the data-source catalogue")
|
|
299
|
+
data_find.add_argument("query", nargs="*", help="Search query")
|
|
300
|
+
data_find.add_argument("--json", action="store_true", help="Print full JSON payload")
|
|
301
|
+
data_find.set_defaults(func=_cmd_data_find)
|
|
302
|
+
|
|
303
|
+
data_show = data_subcommands.add_parser(
|
|
304
|
+
"show", help="Show one source: SKILL.md playbook followed by meta JSON"
|
|
305
|
+
)
|
|
306
|
+
data_show.add_argument("source_id")
|
|
307
|
+
data_show.set_defaults(func=_cmd_data_show)
|
|
308
|
+
|
|
309
|
+
data_query = data_subcommands.add_parser("query", help="Run a source query")
|
|
310
|
+
_add_query_arguments(data_query)
|
|
311
|
+
|
|
312
|
+
query = subcommands.add_parser("query", help="Universal agent-facing source query")
|
|
313
|
+
_add_query_arguments(query)
|
|
314
|
+
|
|
315
|
+
tools = subcommands.add_parser("tools", help="OSINT Navigator tool catalogue (thin API client)")
|
|
316
|
+
tools_subcommands = tools.add_subparsers(dest="tools_command", required=True)
|
|
317
|
+
tools_find = tools_subcommands.add_parser("find", help="Search OSINT tools by meaning")
|
|
318
|
+
tools_find.add_argument("query", nargs="+", help="Search query")
|
|
319
|
+
tools_find.add_argument("--category", help="Optional category filter")
|
|
320
|
+
tools_find.add_argument("--limit", type=int, default=10)
|
|
321
|
+
tools_find.add_argument("--json", action="store_true", help="Print full JSON payload")
|
|
322
|
+
tools_find.set_defaults(func=_cmd_tools_find)
|
|
323
|
+
tools_show = tools_subcommands.add_parser("show", help="Full record + documentation for one tool")
|
|
324
|
+
tools_show.add_argument("tool_id")
|
|
325
|
+
tools_show.add_argument("--json", action="store_true", help="Print full JSON payload")
|
|
326
|
+
tools_show.set_defaults(func=_cmd_tools_show)
|
|
327
|
+
|
|
328
|
+
auth = subcommands.add_parser("auth", help="Membership login for the Navigator CLI")
|
|
329
|
+
auth_subcommands = auth.add_subparsers(dest="auth_command", required=True)
|
|
330
|
+
auth_login = auth_subcommands.add_parser("login", help="Sign in via magic link; stores a PAT in the OS keychain")
|
|
331
|
+
auth_login.add_argument("email", nargs="?", help="Email on your membership")
|
|
332
|
+
auth_login.set_defaults(func=_cmd_auth_login)
|
|
333
|
+
auth_status = auth_subcommands.add_parser("status", help="Show login + quota state")
|
|
334
|
+
auth_status.set_defaults(func=_cmd_auth_status)
|
|
335
|
+
auth_logout = auth_subcommands.add_parser("logout", help="Remove the stored PAT")
|
|
336
|
+
auth_logout.set_defaults(func=_cmd_auth_logout)
|
|
337
|
+
|
|
338
|
+
keys = subcommands.add_parser("keys", help="BYO key helper commands")
|
|
339
|
+
keys_subcommands = keys.add_subparsers(dest="keys_command", required=True)
|
|
340
|
+
keys_list = keys_subcommands.add_parser("list", help="List BYO key configuration state")
|
|
341
|
+
keys_list.set_defaults(func=_cmd_keys_list)
|
|
342
|
+
keys_set = keys_subcommands.add_parser("set", help="Store a BYO API key in the OS keychain")
|
|
343
|
+
keys_set.add_argument("name")
|
|
344
|
+
keys_set.add_argument("--stdin", action="store_true", help="Read the secret value from stdin")
|
|
345
|
+
keys_set.set_defaults(func=_cmd_keys_set)
|
|
346
|
+
keys_get = keys_subcommands.add_parser("get", help="Reveal a configured BYO API key")
|
|
347
|
+
keys_get.add_argument("name")
|
|
348
|
+
keys_get.add_argument("--reveal", action="store_true", help="Required to print the secret")
|
|
349
|
+
keys_get.set_defaults(func=_cmd_keys_get)
|
|
350
|
+
keys_unset = keys_subcommands.add_parser("unset", help="Delete a BYO API key from the OS keychain")
|
|
351
|
+
keys_unset.add_argument("name")
|
|
352
|
+
keys_unset.set_defaults(func=_cmd_keys_unset)
|
|
353
|
+
keys_missing = keys_subcommands.add_parser("missing", help="List sources that require BYO keys")
|
|
354
|
+
keys_missing.set_defaults(func=_cmd_keys_missing)
|
|
355
|
+
|
|
356
|
+
return parser
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
360
|
+
parser = build_parser()
|
|
361
|
+
args = parser.parse_args(argv)
|
|
362
|
+
try:
|
|
363
|
+
return int(args.func(args))
|
|
364
|
+
except ValueError as exc:
|
|
365
|
+
print(f"navigator: {exc}", file=sys.stderr)
|
|
366
|
+
return 2
|
|
367
|
+
except KeyStoreError as exc:
|
|
368
|
+
print(f"navigator: {exc}", file=sys.stderr)
|
|
369
|
+
return 2
|
|
370
|
+
except RuntimeError as exc:
|
|
371
|
+
print(f"navigator: {exc}", file=sys.stderr)
|
|
372
|
+
return 1
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
if __name__ == "__main__":
|
|
376
|
+
raise SystemExit(main())
|