argus-cortex 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus_cortex-0.2.0/.env.example +21 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/.gitignore +3 -1
- argus_cortex-0.2.0/PKG-INFO +142 -0
- argus_cortex-0.2.0/README.md +104 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/pyproject.toml +20 -1
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/__init__.py +47 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/_version.py +2 -2
- argus_cortex-0.2.0/src/argus_cortex/server.py +155 -0
- argus_cortex-0.2.0/src/argus_cortex/store/__init__.py +72 -0
- argus_cortex-0.2.0/src/argus_cortex/store/blob.py +310 -0
- argus_cortex-0.2.0/src/argus_cortex/store/config.py +76 -0
- argus_cortex-0.2.0/src/argus_cortex/store/errors.py +82 -0
- argus_cortex-0.2.0/src/argus_cortex/store/lineage.py +447 -0
- argus_cortex-0.2.0/src/argus_cortex/store/models.py +116 -0
- argus_cortex-0.2.0/src/argus_cortex/store/schema.py +79 -0
- argus_cortex-0.2.0/src/argus_cortex/store/vector.py +316 -0
- argus_cortex-0.2.0/tests/test_blob.py +311 -0
- argus_cortex-0.2.0/tests/test_server.py +151 -0
- argus_cortex-0.2.0/tests/test_store.py +419 -0
- argus_cortex-0.2.0/tests/test_store_pg_integration.py +93 -0
- argus_cortex-0.2.0/tests/test_vector.py +335 -0
- argus_cortex-0.1.0/PKG-INFO +0 -72
- argus_cortex-0.1.0/README.md +0 -47
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/.copier-answers.yml +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/.github/workflows/ci.yml +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/.github/workflows/release.yml +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/LICENSE +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/Makefile +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/backends.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/py.typed +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/taxonomy.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/src/argus_cortex/wire.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/tests/test_backends.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/tests/test_smoke.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/tests/test_taxonomy.py +0 -0
- {argus_cortex-0.1.0 → argus_cortex-0.2.0}/tests/test_wire.py +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Optional stateful services layer (argus_cortex.store).
|
|
2
|
+
# Every value is optional: unset a URL and that feature no-ops (graceful degrade).
|
|
3
|
+
# Copy to .env and fill in only the services you actually run.
|
|
4
|
+
|
|
5
|
+
# Postgres — lineage DAG + human-edit capture (Phase 1).
|
|
6
|
+
# Unset -> open_lineage_store() returns a no-op NullLineageStore.
|
|
7
|
+
CORTEX_PG_URL=postgresql://user:pass@localhost:5432/argus
|
|
8
|
+
# Connection-pool bounds (optional; defaults 0 / 8 — min 0 holds no idle conn).
|
|
9
|
+
CORTEX_PG_POOL_MIN_SIZE=0
|
|
10
|
+
CORTEX_PG_POOL_MAX_SIZE=8
|
|
11
|
+
|
|
12
|
+
# Qdrant — image + tag-set embeddings (Phase 2).
|
|
13
|
+
CORTEX_QDRANT_URL=http://localhost:6333
|
|
14
|
+
|
|
15
|
+
# MinIO / S3 — owned image bytes (Phase 3). Scheme-less endpoint defaults to TLS;
|
|
16
|
+
# write http:// explicitly for a plaintext endpoint. Region matters on AWS S3.
|
|
17
|
+
CORTEX_S3_ENDPOINT=http://localhost:9000
|
|
18
|
+
CORTEX_S3_ACCESS_KEY=
|
|
19
|
+
CORTEX_S3_SECRET_KEY=
|
|
20
|
+
CORTEX_S3_BUCKET=argus
|
|
21
|
+
CORTEX_S3_REGION=us-east-1
|
|
@@ -6,5 +6,7 @@ build/
|
|
|
6
6
|
*.egg-info/
|
|
7
7
|
.pytest_cache/
|
|
8
8
|
.ruff_cache/
|
|
9
|
+
# local service config for the optional stores (.env.example stays tracked)
|
|
10
|
+
.env
|
|
9
11
|
# hatch-vcs writes this at build time (version derived from git tags)
|
|
10
|
-
src/
|
|
12
|
+
src/argus_cortex/_version.py
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: argus-cortex
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Shared foundation for the Argus suite: taxonomy, versioned wire-schema tooling, and the local/remote AI backend contract
|
|
5
|
+
Project-URL: Repository, https://github.com/smk762/argus-cortex
|
|
6
|
+
Author: smk762
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: pydantic>=2.7
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
20
|
+
Requires-Dist: minio>=7.2; extra == 'dev'
|
|
21
|
+
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: qdrant-client>=1.10; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff==0.15.16; extra == 'dev'
|
|
26
|
+
Requires-Dist: starlette>=0.37; extra == 'dev'
|
|
27
|
+
Provides-Extra: postgres
|
|
28
|
+
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'postgres'
|
|
29
|
+
Provides-Extra: qdrant
|
|
30
|
+
Requires-Dist: qdrant-client>=1.10; extra == 'qdrant'
|
|
31
|
+
Provides-Extra: remote
|
|
32
|
+
Requires-Dist: httpx>=0.27; extra == 'remote'
|
|
33
|
+
Provides-Extra: s3
|
|
34
|
+
Requires-Dist: minio>=7.2; extra == 's3'
|
|
35
|
+
Provides-Extra: server
|
|
36
|
+
Requires-Dist: starlette>=0.37; extra == 'server'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# argus-cortex
|
|
40
|
+
|
|
41
|
+
Shared foundation for the Argus suite: taxonomy, versioned wire-schema tooling, and the local/remote AI backend contract.
|
|
42
|
+
|
|
43
|
+
Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (quarry → curator → lens → forge → proof). This is the library the stages import so the code they *share* — the target taxonomy, the wire-schema versioning discipline, and the way they call local or hosted models — lives in one place instead of being copy-pasted per repo.
|
|
44
|
+
|
|
45
|
+
## What's here
|
|
46
|
+
|
|
47
|
+
- **`argus_cortex.taxonomy`** — `TargetProfile` / `TargetStyle` / `TargetCategory`, the "moat" types every stage inherits verbatim.
|
|
48
|
+
- **`argus_cortex.wire`** — the versioned wire-schema toolkit: `check_version()` (major-compatibility gate), `make_versioned_base()` (a Pydantic base that stamps + checks a version field like `proof_version` / `manifest_version`), and `wire_schema()` / `render_schema()` for the committed-schema `schema --check` CLI pattern.
|
|
49
|
+
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
50
|
+
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
51
|
+
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
52
|
+
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
53
|
+
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
54
|
+
- **Phase 3 — MinIO/S3 blob store.** Owns image bytes when cortex can't rely on the filesystem/Immich (exported/selected training images). Blobs are content-addressed on sha256 (`content_key()` == `source_asset.sha256`), so storage, dedup, and lineage join share one identity. `open_blob_store()` no-ops when `CORTEX_S3_ENDPOINT`/`CORTEX_S3_BUCKET` are unset; the `minio` client lives behind the `[s3]` extra.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from argus_cortex.wire import make_versioned_base, render_schema
|
|
58
|
+
from argus_cortex.backends import RemoteBackend, RemoteProvider
|
|
59
|
+
|
|
60
|
+
# per-package versioned base (readable field name kept local, logic shared)
|
|
61
|
+
_Versioned = make_versioned_base("proof_version", "1.0", ("1",))
|
|
62
|
+
|
|
63
|
+
# reach a hosted scorer by IP/port, or a known provider
|
|
64
|
+
scorer = RemoteBackend.from_host("192.168.1.20", 9000, api_key="…")
|
|
65
|
+
nim = RemoteBackend.from_provider(RemoteProvider.NVIDIA_NIM, api_key="…")
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
70
|
+
|
|
71
|
+
# no-op if CORTEX_PG_URL is unset; PostgresLineageStore if it's set
|
|
72
|
+
store = open_lineage_store() # reads CORTEX_* from the environment
|
|
73
|
+
store.ensure_schema() # idempotent bootstrap of the lineage tables
|
|
74
|
+
|
|
75
|
+
# one atomic unit — the asset + caption + edit commit together or not at all
|
|
76
|
+
with store.transaction():
|
|
77
|
+
asset_id = store.record_asset(SourceAsset(uri="immich://…", sha256="…"))
|
|
78
|
+
# lens emits a CaptionResult; cortex ingests it (cortex never imports lens)
|
|
79
|
+
cap_id = store.record_caption(asset_id, Caption.from_caption_result(result, profile={...}))
|
|
80
|
+
store.record_edit(cap_id, HumanEdit(edited_caption="…", editor="alice"))
|
|
81
|
+
|
|
82
|
+
# the (model_caption, edited_caption) pairs the summariser feedback loop trains on
|
|
83
|
+
pairs = store.caption_edit_pairs()
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from argus_cortex.store import open_vector_store, IMAGE_COLLECTION
|
|
88
|
+
|
|
89
|
+
# no-op if CORTEX_QDRANT_URL is unset; QdrantVectorStore if it's set
|
|
90
|
+
vec = open_vector_store()
|
|
91
|
+
vec.ensure_collection(IMAGE_COLLECTION, dim=512) # idempotent
|
|
92
|
+
|
|
93
|
+
# store an embedding you computed, tagged with its lineage id
|
|
94
|
+
vec.upsert(IMAGE_COLLECTION, point_id=cap_id, vector=embedding, payload={"caption_id": cap_id})
|
|
95
|
+
|
|
96
|
+
# retrieve similar past captions to steer the summariser (few-shot)
|
|
97
|
+
hits = vec.search(IMAGE_COLLECTION, vector=query_embedding, top_k=5)
|
|
98
|
+
similar_caption_ids = [h.payload["caption_id"] for h in hits] # join back to Postgres
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from argus_cortex.store import open_blob_store
|
|
103
|
+
|
|
104
|
+
# no-op if CORTEX_S3_ENDPOINT / CORTEX_S3_BUCKET are unset; S3BlobStore if set
|
|
105
|
+
blobs = open_blob_store()
|
|
106
|
+
blobs.ensure_bucket()
|
|
107
|
+
|
|
108
|
+
# content-addressed: the returned key IS the sha256 (== source_asset.sha256)
|
|
109
|
+
sha = blobs.put_content_addressed(image_bytes, content_type="image/jpeg")
|
|
110
|
+
data = blobs.get(sha) # bytes, or None if absent
|
|
111
|
+
link = blobs.url(sha, expires=3600) # presigned GET URL
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Install
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
uv pip install argus-cortex # core (pydantic only)
|
|
118
|
+
uv pip install "argus-cortex[remote]" # + httpx for RemoteBackend
|
|
119
|
+
uv pip install "argus-cortex[server]" # + starlette for argus_cortex.server (WriteGuard, env_flag)
|
|
120
|
+
uv pip install "argus-cortex[postgres]" # + psycopg for the lineage store
|
|
121
|
+
uv pip install "argus-cortex[qdrant]" # + qdrant-client for the vector store
|
|
122
|
+
uv pip install "argus-cortex[s3]" # + minio for the blob store
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The stateful services are configured via `CORTEX_*` env vars — see [`.env.example`](.env.example). Each is optional: leave a URL unset and that feature degrades to a no-op.
|
|
126
|
+
|
|
127
|
+
## Develop
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
make install # venv + editable install with the "dev" extras
|
|
131
|
+
make test
|
|
132
|
+
make lint
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## CI / Release
|
|
136
|
+
|
|
137
|
+
- **CI** runs via the shared [`argus-ci`](https://github.com/smk762/argus-ci) reusable workflow.
|
|
138
|
+
- **Release** publishes to PyPI (OIDC trusted publishing) on `v*` tags. No container image — this is a library.
|
|
139
|
+
- Versioning is derived from git tags via `hatch-vcs` — tag `vX.Y.Z` to cut a release.
|
|
140
|
+
|
|
141
|
+
This repo was scaffolded from [`argus-pkg-template`](https://github.com/smk762/argus-pkg-template).
|
|
142
|
+
Run `copier update` to pull template changes (CI, release, tooling).
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# argus-cortex
|
|
2
|
+
|
|
3
|
+
Shared foundation for the Argus suite: taxonomy, versioned wire-schema tooling, and the local/remote AI backend contract.
|
|
4
|
+
|
|
5
|
+
Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (quarry → curator → lens → forge → proof). This is the library the stages import so the code they *share* — the target taxonomy, the wire-schema versioning discipline, and the way they call local or hosted models — lives in one place instead of being copy-pasted per repo.
|
|
6
|
+
|
|
7
|
+
## What's here
|
|
8
|
+
|
|
9
|
+
- **`argus_cortex.taxonomy`** — `TargetProfile` / `TargetStyle` / `TargetCategory`, the "moat" types every stage inherits verbatim.
|
|
10
|
+
- **`argus_cortex.wire`** — the versioned wire-schema toolkit: `check_version()` (major-compatibility gate), `make_versioned_base()` (a Pydantic base that stamps + checks a version field like `proof_version` / `manifest_version`), and `wire_schema()` / `render_schema()` for the committed-schema `schema --check` CLI pattern.
|
|
11
|
+
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
12
|
+
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
13
|
+
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
14
|
+
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
15
|
+
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
16
|
+
- **Phase 3 — MinIO/S3 blob store.** Owns image bytes when cortex can't rely on the filesystem/Immich (exported/selected training images). Blobs are content-addressed on sha256 (`content_key()` == `source_asset.sha256`), so storage, dedup, and lineage join share one identity. `open_blob_store()` no-ops when `CORTEX_S3_ENDPOINT`/`CORTEX_S3_BUCKET` are unset; the `minio` client lives behind the `[s3]` extra.
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from argus_cortex.wire import make_versioned_base, render_schema
|
|
20
|
+
from argus_cortex.backends import RemoteBackend, RemoteProvider
|
|
21
|
+
|
|
22
|
+
# per-package versioned base (readable field name kept local, logic shared)
|
|
23
|
+
_Versioned = make_versioned_base("proof_version", "1.0", ("1",))
|
|
24
|
+
|
|
25
|
+
# reach a hosted scorer by IP/port, or a known provider
|
|
26
|
+
scorer = RemoteBackend.from_host("192.168.1.20", 9000, api_key="…")
|
|
27
|
+
nim = RemoteBackend.from_provider(RemoteProvider.NVIDIA_NIM, api_key="…")
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
32
|
+
|
|
33
|
+
# no-op if CORTEX_PG_URL is unset; PostgresLineageStore if it's set
|
|
34
|
+
store = open_lineage_store() # reads CORTEX_* from the environment
|
|
35
|
+
store.ensure_schema() # idempotent bootstrap of the lineage tables
|
|
36
|
+
|
|
37
|
+
# one atomic unit — the asset + caption + edit commit together or not at all
|
|
38
|
+
with store.transaction():
|
|
39
|
+
asset_id = store.record_asset(SourceAsset(uri="immich://…", sha256="…"))
|
|
40
|
+
# lens emits a CaptionResult; cortex ingests it (cortex never imports lens)
|
|
41
|
+
cap_id = store.record_caption(asset_id, Caption.from_caption_result(result, profile={...}))
|
|
42
|
+
store.record_edit(cap_id, HumanEdit(edited_caption="…", editor="alice"))
|
|
43
|
+
|
|
44
|
+
# the (model_caption, edited_caption) pairs the summariser feedback loop trains on
|
|
45
|
+
pairs = store.caption_edit_pairs()
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from argus_cortex.store import open_vector_store, IMAGE_COLLECTION
|
|
50
|
+
|
|
51
|
+
# no-op if CORTEX_QDRANT_URL is unset; QdrantVectorStore if it's set
|
|
52
|
+
vec = open_vector_store()
|
|
53
|
+
vec.ensure_collection(IMAGE_COLLECTION, dim=512) # idempotent
|
|
54
|
+
|
|
55
|
+
# store an embedding you computed, tagged with its lineage id
|
|
56
|
+
vec.upsert(IMAGE_COLLECTION, point_id=cap_id, vector=embedding, payload={"caption_id": cap_id})
|
|
57
|
+
|
|
58
|
+
# retrieve similar past captions to steer the summariser (few-shot)
|
|
59
|
+
hits = vec.search(IMAGE_COLLECTION, vector=query_embedding, top_k=5)
|
|
60
|
+
similar_caption_ids = [h.payload["caption_id"] for h in hits] # join back to Postgres
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from argus_cortex.store import open_blob_store
|
|
65
|
+
|
|
66
|
+
# no-op if CORTEX_S3_ENDPOINT / CORTEX_S3_BUCKET are unset; S3BlobStore if set
|
|
67
|
+
blobs = open_blob_store()
|
|
68
|
+
blobs.ensure_bucket()
|
|
69
|
+
|
|
70
|
+
# content-addressed: the returned key IS the sha256 (== source_asset.sha256)
|
|
71
|
+
sha = blobs.put_content_addressed(image_bytes, content_type="image/jpeg")
|
|
72
|
+
data = blobs.get(sha) # bytes, or None if absent
|
|
73
|
+
link = blobs.url(sha, expires=3600) # presigned GET URL
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Install
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
uv pip install argus-cortex # core (pydantic only)
|
|
80
|
+
uv pip install "argus-cortex[remote]" # + httpx for RemoteBackend
|
|
81
|
+
uv pip install "argus-cortex[server]" # + starlette for argus_cortex.server (WriteGuard, env_flag)
|
|
82
|
+
uv pip install "argus-cortex[postgres]" # + psycopg for the lineage store
|
|
83
|
+
uv pip install "argus-cortex[qdrant]" # + qdrant-client for the vector store
|
|
84
|
+
uv pip install "argus-cortex[s3]" # + minio for the blob store
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The stateful services are configured via `CORTEX_*` env vars — see [`.env.example`](.env.example). Each is optional: leave a URL unset and that feature degrades to a no-op.
|
|
88
|
+
|
|
89
|
+
## Develop
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
make install # venv + editable install with the "dev" extras
|
|
93
|
+
make test
|
|
94
|
+
make lint
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## CI / Release
|
|
98
|
+
|
|
99
|
+
- **CI** runs via the shared [`argus-ci`](https://github.com/smk762/argus-ci) reusable workflow.
|
|
100
|
+
- **Release** publishes to PyPI (OIDC trusted publishing) on `v*` tags. No container image — this is a library.
|
|
101
|
+
- Versioning is derived from git tags via `hatch-vcs` — tag `vX.Y.Z` to cut a release.
|
|
102
|
+
|
|
103
|
+
This repo was scaffolded from [`argus-pkg-template`](https://github.com/smk762/argus-pkg-template).
|
|
104
|
+
Run `copier update` to pull template changes (CI, release, tooling).
|
|
@@ -25,7 +25,26 @@ dependencies = [
|
|
|
25
25
|
|
|
26
26
|
[project.optional-dependencies]
|
|
27
27
|
remote = ["httpx>=0.27"]
|
|
28
|
-
|
|
28
|
+
postgres = ["psycopg[binary,pool]>=3.1"]
|
|
29
|
+
qdrant = ["qdrant-client>=1.10"]
|
|
30
|
+
s3 = ["minio>=7.2"]
|
|
31
|
+
# The shared ASGI write-guard + env-flag scaffolding for the suite's servers
|
|
32
|
+
# (argus_cortex.server). Starlette ships with FastAPI, so consumers already have
|
|
33
|
+
# it via their own [server] extra.
|
|
34
|
+
server = ["starlette>=0.37"]
|
|
35
|
+
# The store drivers are in dev so each store's real client path (pool/enum/client
|
|
36
|
+
# construction, error wrapping) is exercised in CI, not just via fakes.
|
|
37
|
+
# starlette + httpx power the pure-ASGI guard tests (Starlette TestClient).
|
|
38
|
+
dev = [
|
|
39
|
+
"pytest>=8.0",
|
|
40
|
+
"pytest-asyncio>=0.23",
|
|
41
|
+
"ruff==0.15.16",
|
|
42
|
+
"psycopg[binary,pool]>=3.1",
|
|
43
|
+
"qdrant-client>=1.10",
|
|
44
|
+
"minio>=7.2",
|
|
45
|
+
"starlette>=0.37",
|
|
46
|
+
"httpx>=0.27",
|
|
47
|
+
]
|
|
29
48
|
|
|
30
49
|
[project.urls]
|
|
31
50
|
Repository = "https://github.com/smk762/argus-cortex"
|
|
@@ -23,6 +23,30 @@ from argus_cortex.backends import (
|
|
|
23
23
|
resolve_device,
|
|
24
24
|
with_retries,
|
|
25
25
|
)
|
|
26
|
+
from argus_cortex.store import (
|
|
27
|
+
IMAGE_COLLECTION,
|
|
28
|
+
TAGSET_COLLECTION,
|
|
29
|
+
BlobStore,
|
|
30
|
+
Caption,
|
|
31
|
+
HumanEdit,
|
|
32
|
+
LineageStore,
|
|
33
|
+
NullBlobStore,
|
|
34
|
+
NullLineageStore,
|
|
35
|
+
NullVectorStore,
|
|
36
|
+
PostgresLineageStore,
|
|
37
|
+
QdrantVectorStore,
|
|
38
|
+
S3BlobStore,
|
|
39
|
+
SourceAsset,
|
|
40
|
+
StoreConfig,
|
|
41
|
+
StoreError,
|
|
42
|
+
TrainingRun,
|
|
43
|
+
VectorHit,
|
|
44
|
+
VectorStore,
|
|
45
|
+
content_key,
|
|
46
|
+
open_blob_store,
|
|
47
|
+
open_lineage_store,
|
|
48
|
+
open_vector_store,
|
|
49
|
+
)
|
|
26
50
|
from argus_cortex.taxonomy import TargetCategory, TargetProfile, TargetStyle
|
|
27
51
|
from argus_cortex.wire import (
|
|
28
52
|
VersionError,
|
|
@@ -55,4 +79,27 @@ __all__ = [
|
|
|
55
79
|
"RemoteProvider",
|
|
56
80
|
"resolve_device",
|
|
57
81
|
"with_retries",
|
|
82
|
+
# store (optional stateful services layer)
|
|
83
|
+
"StoreConfig",
|
|
84
|
+
"StoreError",
|
|
85
|
+
"LineageStore",
|
|
86
|
+
"NullLineageStore",
|
|
87
|
+
"PostgresLineageStore",
|
|
88
|
+
"open_lineage_store",
|
|
89
|
+
"Caption",
|
|
90
|
+
"HumanEdit",
|
|
91
|
+
"SourceAsset",
|
|
92
|
+
"TrainingRun",
|
|
93
|
+
"VectorStore",
|
|
94
|
+
"NullVectorStore",
|
|
95
|
+
"QdrantVectorStore",
|
|
96
|
+
"VectorHit",
|
|
97
|
+
"open_vector_store",
|
|
98
|
+
"IMAGE_COLLECTION",
|
|
99
|
+
"TAGSET_COLLECTION",
|
|
100
|
+
"BlobStore",
|
|
101
|
+
"NullBlobStore",
|
|
102
|
+
"S3BlobStore",
|
|
103
|
+
"open_blob_store",
|
|
104
|
+
"content_key",
|
|
58
105
|
]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Shared ASGI write-guard + env-flag scaffolding for the suite's micro-servers.
|
|
2
|
+
|
|
3
|
+
Every Argus micro-server (proof, curator, forge, lens) is a small FastAPI app
|
|
4
|
+
that must fence off state-changing requests in some deployment mode:
|
|
5
|
+
|
|
6
|
+
* **read-only / replay** (argus-proof #45) — a GPU-less public demo keeps serving
|
|
7
|
+
stored reports but refuses every write and every live-eval trigger.
|
|
8
|
+
* **cross-site protection** (argus-curator #3) — CORS is not a write boundary, so
|
|
9
|
+
unsafe methods carrying an untrusted ``Origin`` are refused before the routes.
|
|
10
|
+
|
|
11
|
+
Both are the same shape: gate the *unsafe* HTTP methods, refuse with a ``403``
|
|
12
|
+
JSON ``{"detail": ...}``, let reads through. :class:`WriteGuard` is that one
|
|
13
|
+
gate, parameterised by a *refuse* predicate; :func:`constant_refuse` and
|
|
14
|
+
:func:`cross_site_refuse` are the two predicates the suite needs today.
|
|
15
|
+
|
|
16
|
+
The guard is deliberately **pure ASGI**, not Starlette's ``BaseHTTPMiddleware``:
|
|
17
|
+
``BaseHTTPMiddleware`` runs the response through an anyio memory stream, which
|
|
18
|
+
buffers the SSE / NDJSON streams these servers return and adds a hop to every
|
|
19
|
+
read on a read-heavy demo host. A pure-ASGI guard forwards the untouched ``send``
|
|
20
|
+
channel, so streaming endpoints stream and reads pay nothing.
|
|
21
|
+
|
|
22
|
+
Starlette ships with FastAPI, so this module lives behind the ``argus-cortex``
|
|
23
|
+
``[server]`` extra and is never imported at ``import argus_cortex`` time.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import logging
|
|
29
|
+
import os
|
|
30
|
+
from collections.abc import Callable, Iterable
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
from starlette.datastructures import Headers
|
|
34
|
+
from starlette.responses import JSONResponse
|
|
35
|
+
from starlette.types import ASGIApp, Receive, Scope, Send
|
|
36
|
+
except ImportError as exc: # pragma: no cover
|
|
37
|
+
raise ImportError("argus_cortex.server requires Starlette: pip install argus-cortex[server]") from exc
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"FALSY",
|
|
41
|
+
"TRUTHY",
|
|
42
|
+
"UNSAFE_METHODS",
|
|
43
|
+
"Refuse",
|
|
44
|
+
"WriteGuard",
|
|
45
|
+
"constant_refuse",
|
|
46
|
+
"cross_site_refuse",
|
|
47
|
+
"env_flag",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
# HTTP methods that change server state (or trigger compute). Idempotent reads
|
|
51
|
+
# (GET/HEAD/OPTIONS) are never gated — they stay CORS's business.
|
|
52
|
+
UNSAFE_METHODS: frozenset[str] = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
|
53
|
+
|
|
54
|
+
# Recognised env-flag spellings. A *set* value in neither set is a
|
|
55
|
+
# misconfiguration (a typo like ``...=enabled``), surfaced by :func:`env_flag`.
|
|
56
|
+
TRUTHY: frozenset[str] = frozenset({"1", "true", "yes", "on"})
|
|
57
|
+
FALSY: frozenset[str] = frozenset({"0", "false", "no", "off", ""})
|
|
58
|
+
|
|
59
|
+
_log = logging.getLogger(__name__)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def env_flag(name: str) -> bool:
|
|
63
|
+
"""Whether env var *name* is set to a truthy string (``1``/``true``/``yes``/``on``).
|
|
64
|
+
|
|
65
|
+
A set-but-unrecognised value (a typo like ``ARGUS_PROOF_READ_ONLY=enabled``)
|
|
66
|
+
logs a warning and is treated as **off**, so a mistyped protection flag is
|
|
67
|
+
visible in the logs rather than silently leaving the guard disabled.
|
|
68
|
+
"""
|
|
69
|
+
raw = os.environ.get(name, "").strip().lower()
|
|
70
|
+
if raw in TRUTHY:
|
|
71
|
+
return True
|
|
72
|
+
if raw not in FALSY:
|
|
73
|
+
_log.warning("env flag %s set to an unrecognised value %r; treating as off", name, raw)
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# A refusal predicate: given the scope + headers of an *unsafe* request, return a
|
|
78
|
+
# ``403`` detail message to refuse it, or ``None`` to let it through.
|
|
79
|
+
Refuse = Callable[[Scope, Headers], "str | None"]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class WriteGuard:
|
|
83
|
+
"""Pure-ASGI gate on state-changing HTTP methods.
|
|
84
|
+
|
|
85
|
+
For every request whose method is in :data:`UNSAFE_METHODS`, consult
|
|
86
|
+
``refuse(scope, headers)``: a returned string becomes a ``403`` JSON
|
|
87
|
+
``{"detail": ...}`` and the wrapped app is never called; ``None`` lets the
|
|
88
|
+
request through. Reads (and every non-``http`` scope) are forwarded untouched.
|
|
89
|
+
|
|
90
|
+
Register it *before* ``CORSMiddleware`` (``add_middleware`` inserts at the top
|
|
91
|
+
of the stack, so the last added is outermost) — that keeps CORS the outer
|
|
92
|
+
layer and lets it still annotate a refused cross-origin write with its
|
|
93
|
+
headers, so the caller sees a readable ``403`` rather than an opaque CORS
|
|
94
|
+
error.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
def __init__(self, app: ASGIApp, refuse: Refuse) -> None:
|
|
98
|
+
self.app = app
|
|
99
|
+
self.refuse = refuse
|
|
100
|
+
|
|
101
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
102
|
+
if scope["type"] == "http" and scope["method"] in UNSAFE_METHODS:
|
|
103
|
+
detail = self.refuse(scope, Headers(scope=scope))
|
|
104
|
+
if detail is not None:
|
|
105
|
+
await JSONResponse({"detail": detail}, status_code=403)(scope, receive, send)
|
|
106
|
+
return
|
|
107
|
+
await self.app(scope, receive, send)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def constant_refuse(message: str) -> Refuse:
|
|
111
|
+
"""A :class:`WriteGuard` predicate refusing *every* unsafe request with the
|
|
112
|
+
same *message* — read-only / replay mode, where no write is ever allowed.
|
|
113
|
+
|
|
114
|
+
Method-based, so it covers every write route — current and future — without
|
|
115
|
+
per-route guards to forget.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
def _refuse(scope: Scope, headers: Headers) -> str | None:
|
|
119
|
+
return message
|
|
120
|
+
|
|
121
|
+
return _refuse
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def cross_site_refuse(trusted_origins: Iterable[str]) -> Refuse:
|
|
125
|
+
"""A :class:`WriteGuard` predicate refusing untrusted cross-origin writes.
|
|
126
|
+
|
|
127
|
+
CORS is not a write boundary: a ``multipart/form-data`` POST is a
|
|
128
|
+
CORS-safelisted content type, so a browser sends it with **no preflight** and
|
|
129
|
+
any page the user visits can drive it — the same-origin policy only stops
|
|
130
|
+
that page from *reading* the reply. For an unauthenticated server commonly
|
|
131
|
+
bound on localhost or a LAN address the attacker's own server cannot reach,
|
|
132
|
+
that is enough to poison data (or move files about). So unsafe methods are
|
|
133
|
+
gated on ``Origin``:
|
|
134
|
+
|
|
135
|
+
* absent -> allowed. Non-browser clients (curl, the CLI, server-to-server)
|
|
136
|
+
never send it, and browsers always do for cross-origin state changes.
|
|
137
|
+
* same host:port as the request -> allowed, so a UI proxied onto this host
|
|
138
|
+
keeps working un-allow-listed. Compared host:port (not full origin) because
|
|
139
|
+
behind a TLS-terminating proxy the app sees ``http://`` while the browser
|
|
140
|
+
reports ``https://`` — same host either way.
|
|
141
|
+
* in *trusted_origins* -> allowed. That is the operator's own allow-list.
|
|
142
|
+
* anything else -> refused.
|
|
143
|
+
"""
|
|
144
|
+
trusted = set(trusted_origins)
|
|
145
|
+
|
|
146
|
+
def _refuse(scope: Scope, headers: Headers) -> str | None:
|
|
147
|
+
origin = headers.get("origin")
|
|
148
|
+
if origin is None or origin in trusted:
|
|
149
|
+
return None
|
|
150
|
+
host = headers.get("host")
|
|
151
|
+
if host and origin.partition("://")[2] == host:
|
|
152
|
+
return None
|
|
153
|
+
return f"cross-site {scope['method']} from origin {origin} is not allowed"
|
|
154
|
+
|
|
155
|
+
return _refuse
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Optional stateful services layer — the suite's persistent "memory".
|
|
2
|
+
|
|
3
|
+
External and opt-in: each store is addressed by a ``CORTEX_*`` URL in the
|
|
4
|
+
environment and no-ops when unset (see :class:`~argus_cortex.store.config.StoreConfig`).
|
|
5
|
+
Phase 1 ships the Postgres lineage store, phase 2 the Qdrant vector store, and
|
|
6
|
+
phase 3 the MinIO/S3 blob store.
|
|
7
|
+
|
|
8
|
+
The dependency direction is ``cortex → lens``: cortex defines the lineage
|
|
9
|
+
entities in :mod:`~argus_cortex.store.models` and the embedding model is the
|
|
10
|
+
caller's concern, so lens never imports cortex and stays DB-free.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from argus_cortex.store.blob import (
|
|
16
|
+
BlobStore,
|
|
17
|
+
NullBlobStore,
|
|
18
|
+
S3BlobStore,
|
|
19
|
+
content_key,
|
|
20
|
+
open_blob_store,
|
|
21
|
+
)
|
|
22
|
+
from argus_cortex.store.config import StoreConfig
|
|
23
|
+
from argus_cortex.store.errors import StoreError
|
|
24
|
+
from argus_cortex.store.lineage import (
|
|
25
|
+
LineageStore,
|
|
26
|
+
NullLineageStore,
|
|
27
|
+
PostgresLineageStore,
|
|
28
|
+
open_lineage_store,
|
|
29
|
+
)
|
|
30
|
+
from argus_cortex.store.models import (
|
|
31
|
+
Caption,
|
|
32
|
+
HumanEdit,
|
|
33
|
+
SourceAsset,
|
|
34
|
+
TrainingRun,
|
|
35
|
+
)
|
|
36
|
+
from argus_cortex.store.vector import (
|
|
37
|
+
IMAGE_COLLECTION,
|
|
38
|
+
TAGSET_COLLECTION,
|
|
39
|
+
NullVectorStore,
|
|
40
|
+
QdrantVectorStore,
|
|
41
|
+
VectorHit,
|
|
42
|
+
VectorStore,
|
|
43
|
+
open_vector_store,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"StoreConfig",
|
|
48
|
+
"StoreError",
|
|
49
|
+
# lineage (phase 1: Postgres)
|
|
50
|
+
"LineageStore",
|
|
51
|
+
"NullLineageStore",
|
|
52
|
+
"PostgresLineageStore",
|
|
53
|
+
"open_lineage_store",
|
|
54
|
+
"Caption",
|
|
55
|
+
"HumanEdit",
|
|
56
|
+
"SourceAsset",
|
|
57
|
+
"TrainingRun",
|
|
58
|
+
# vector (phase 2: Qdrant)
|
|
59
|
+
"VectorStore",
|
|
60
|
+
"NullVectorStore",
|
|
61
|
+
"QdrantVectorStore",
|
|
62
|
+
"VectorHit",
|
|
63
|
+
"open_vector_store",
|
|
64
|
+
"IMAGE_COLLECTION",
|
|
65
|
+
"TAGSET_COLLECTION",
|
|
66
|
+
# blob (phase 3: MinIO/S3)
|
|
67
|
+
"BlobStore",
|
|
68
|
+
"NullBlobStore",
|
|
69
|
+
"S3BlobStore",
|
|
70
|
+
"open_blob_store",
|
|
71
|
+
"content_key",
|
|
72
|
+
]
|