densemem 2.7.0__tar.gz → 2.7.2rc4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {densemem-2.7.0 → densemem-2.7.2rc4}/PKG-INFO +41 -56
- {densemem-2.7.0 → densemem-2.7.2rc4}/README.md +39 -54
- {densemem-2.7.0 → densemem-2.7.2rc4}/examples/remember.py +4 -14
- {densemem-2.7.0 → densemem-2.7.2rc4}/pyproject.toml +2 -2
- densemem-2.7.2rc4/scripts/audit.py +160 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/scripts/matrix.sh +14 -3
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/__init__.py +4 -3
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/client.py +10 -2
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/langchain.py +4 -3
- densemem-2.7.2rc4/src/densemem/requests.py +62 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/results.py +29 -35
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/conftest.py +3 -1
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/e2e.py +46 -57
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/e2e_adk.py +3 -4
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/e2e_langchain.py +9 -11
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/test_adk.py +21 -5
- densemem-2.7.2rc4/tests/test_audit.py +282 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/test_langchain.py +30 -25
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/test_transport.py +40 -17
- densemem-2.7.0/scripts/audit.py +0 -90
- densemem-2.7.0/src/densemem/requests.py +0 -116
- densemem-2.7.0/tests/test_audit.py +0 -115
- {densemem-2.7.0 → densemem-2.7.2rc4}/.gitignore +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/LICENSE +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/examples/adk_memory.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/examples/langchain_retriever.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/examples/session.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/requirements-dev.txt +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/scripts/check.sh +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/scripts/e2e.sh +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/scripts/release.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/adk.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/errors.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/src/densemem/py.typed +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/e2e_support.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/test_release.py +0 -0
- {densemem-2.7.0 → densemem-2.7.2rc4}/tests/test_release_propagation.py +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: densemem
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.2rc4
|
|
4
4
|
Summary: Typed async and sync clients for Dense-Mem MCP
|
|
5
|
-
Project-URL: Source, https://github.com/markhuangai/dense-mem/tree/
|
|
5
|
+
Project-URL: Source, https://github.com/markhuangai/dense-mem/tree/73080b340db41db481837f9ebd0ee6151321a96f
|
|
6
6
|
Project-URL: Repository, https://github.com/markhuangai/dense-mem
|
|
7
7
|
Project-URL: Documentation, https://github.com/markhuangai/dense-mem/wiki/Framework-Integrations
|
|
8
8
|
Project-URL: Issues, https://github.com/markhuangai/dense-mem/issues
|
|
@@ -61,9 +61,8 @@ use inside an active event loop.
|
|
|
61
61
|
Methods are initialize, list_tools, remember, ingest_session, recall_memory and
|
|
62
62
|
trace_memory. Requests/results are typed dictionaries with py.typed. Server
|
|
63
63
|
validation and authenticated discovery remain authoritative. Current-server
|
|
64
|
-
compatibility is dense-mem.v2.
|
|
65
|
-
|
|
66
|
-
Historical requests without a target fail validation, including retries.
|
|
64
|
+
compatibility is dense-mem.v2.8.0. Refresh tool discovery after upgrading.
|
|
65
|
+
Retired proposal fields fail validation before provider work.
|
|
67
66
|
|
|
68
67
|
## Complete Remember request and safe replay
|
|
69
68
|
|
|
@@ -73,45 +72,31 @@ from densemem import RememberRequest
|
|
|
73
72
|
request: RememberRequest = {
|
|
74
73
|
"target_space": "team_shared",
|
|
75
74
|
"idempotency_key": retained_operation_key,
|
|
76
|
-
"evidence": [
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
83
|
-
],
|
|
84
|
-
"relationships": [
|
|
85
|
-
{
|
|
86
|
-
"ref": "durable-store",
|
|
87
|
-
"subject": {"name": "Dense-Mem", "entity_kind": "project"},
|
|
88
|
-
"predicate": {"proposed_key": "stores_durable_memory_in"},
|
|
89
|
-
"object": {"entity": {"name": "PostgreSQL", "entity_kind": "product"}},
|
|
90
|
-
"polarity": "+",
|
|
91
|
-
"evidence_indices": [0],
|
|
92
|
-
}
|
|
93
|
-
],
|
|
75
|
+
"evidence": [{
|
|
76
|
+
"content": "Dense-Mem stores durable memory in PostgreSQL.",
|
|
77
|
+
"source_type": "manual",
|
|
78
|
+
"source": "Python core example",
|
|
79
|
+
"metadata": {"origin": "python-example"},
|
|
80
|
+
}],
|
|
94
81
|
}
|
|
95
82
|
result = await client.remember(request)
|
|
96
83
|
~~~
|
|
97
84
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
one proposal citing every evidence item. The client does not infer Relationships.
|
|
85
|
+
Retain the complete request and key before sending. Replay the unchanged request
|
|
86
|
+
including target, metadata, and key to reconcile an unknown outcome. The client
|
|
87
|
+
never retries automatically or invents defaults. Legacy operation keys return
|
|
88
|
+
non-retryable retired_contract guidance without duplicating accepted Evidence.
|
|
103
89
|
|
|
104
|
-
Choose private for personal
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
90
|
+
Choose private for personal memory and team_shared for team knowledge.
|
|
91
|
+
Authentication selects the actual team, owner, and space. Every credential has
|
|
92
|
+
separate private memory. Team-read-only credentials may remember privately;
|
|
93
|
+
shared writes require current permission. The server never redirects a denied
|
|
94
|
+
shared write.
|
|
109
95
|
|
|
110
|
-
Inspect
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
provenance
|
|
114
|
-
establish new factual support. Trace requires a returned relationship_id.
|
|
96
|
+
Inspect Evidence outcomes, search states, and explicit warnings. Recall preserves
|
|
97
|
+
server order, equivalent IDs, Evidence conflicts, topic hints, and degradations.
|
|
98
|
+
Trace takes an evidence_id. Export accepts 1–500 Evidence IDs and returns a
|
|
99
|
+
versioned Evidence/provenance artifact with hashes and omission reporting.
|
|
115
100
|
|
|
116
101
|
## Exact private session ingestion
|
|
117
102
|
|
|
@@ -141,9 +126,10 @@ never authorization or private-space selection. The server must enable session
|
|
|
141
126
|
ingestion and the caller must have effective private write access. Every active
|
|
142
127
|
credential has its own private space, including team-read-only credentials;
|
|
143
128
|
SSO sessions use their user/team membership's separate private space. Discovery
|
|
144
|
-
hides unavailable tools. Remember requires its explicit target
|
|
145
|
-
Session
|
|
146
|
-
|
|
129
|
+
hides unavailable tools. Remember requires its explicit target.
|
|
130
|
+
Session acceptance is atomic across the request. The server selects exact
|
|
131
|
+
current-event excerpts with original offsets using session_evidence_excerpts_v2.
|
|
132
|
+
Empty extraction returns memory_outcome no_memory with search_state not_required.
|
|
147
133
|
|
|
148
134
|
## Failures and unknown outcomes
|
|
149
135
|
|
|
@@ -249,15 +235,16 @@ version matrix, and scripts/e2e.sh python_client for real server boundaries.
|
|
|
249
235
|
The example scripts require DENSEMEM_ENDPOINT, DENSEMEM_TOKEN and a retained
|
|
250
236
|
DENSEMEM_OPERATION_KEY. Keep the exact request unchanged when rerunning them.
|
|
251
237
|
|
|
252
|
-
The
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
238
|
+
The root [dependency exception policy](../../dependency-exceptions.json) is the
|
|
239
|
+
sole source of approved Go and Python vulnerability exceptions. It records exact
|
|
240
|
+
advisories, dependencies, versions, ADK prerequisites, reasons, approval links,
|
|
241
|
+
and exclusive UTC expiry dates. Both scanners resolve it from their repository
|
|
242
|
+
location, independently of the working directory; environment variables cannot
|
|
243
|
+
replace approvals. Missing or invalid policy fails closed. Empty sections grant
|
|
244
|
+
no exceptions. Scans still run and retain complete reports in output files or
|
|
245
|
+
console logs. Other findings, skipped dependencies and scanner failures remain
|
|
246
|
+
blocking, and enforcement resumes automatically at expiry. Remove an exception
|
|
247
|
+
sooner when compatible dependencies support its patched version.
|
|
261
248
|
|
|
262
249
|
The server prerelease and stable-release workflows call publish-python to build
|
|
263
250
|
and verify artifacts after their release gates pass, then upload through their own
|
|
@@ -318,9 +305,7 @@ or credential logging; caller-configured LangChain callbacks receive the normal
|
|
|
318
305
|
queries and returned Documents.
|
|
319
306
|
|
|
320
307
|
k defaults to 10 and accepts only integers from 1 through 50. recall_options
|
|
321
|
-
accepts
|
|
322
|
-
community_relationship_limit, valid_at, known_at, known_evidence_ids,
|
|
323
|
-
known_relationship_ids and expand_from_entity_ids. Values pass to the server
|
|
308
|
+
accepts topic_ids, topic_limit, valid_at, known_at, and known_evidence_ids. Values pass to the server
|
|
324
309
|
unchanged and are validated there. Query, limit, team and space overrides are
|
|
325
310
|
rejected; invocation-time recall overrides are unsupported. Options are copied
|
|
326
311
|
for each call. Server validation and authorization remain authoritative.
|
|
@@ -329,9 +314,9 @@ Each returned evidence item becomes one Document in server order. Document.id
|
|
|
329
314
|
is its evidence_id, and page_content is the exact server-bounded context without
|
|
330
315
|
additional truncation. Document.metadata["densemem"] contains evidence_id,
|
|
331
316
|
equivalent_evidence_ids, equivalents_truncated, space_kind, recall_id,
|
|
332
|
-
|
|
333
|
-
and suggested_actions.
|
|
334
|
-
|
|
317
|
+
related_topics, topics_truncated, conflicts, search_states, degradations
|
|
318
|
+
and suggested_actions. Topic hints describe the query and support bounded
|
|
319
|
+
follow-up through 1–10 unique topic_ids without descendant expansion. The current public
|
|
335
320
|
response exposes no document scores, ranks or source URLs, so none are invented.
|
|
336
321
|
Identical private and shared text remains separate Documents.
|
|
337
322
|
|
|
@@ -36,9 +36,8 @@ use inside an active event loop.
|
|
|
36
36
|
Methods are initialize, list_tools, remember, ingest_session, recall_memory and
|
|
37
37
|
trace_memory. Requests/results are typed dictionaries with py.typed. Server
|
|
38
38
|
validation and authenticated discovery remain authoritative. Current-server
|
|
39
|
-
compatibility is dense-mem.v2.
|
|
40
|
-
|
|
41
|
-
Historical requests without a target fail validation, including retries.
|
|
39
|
+
compatibility is dense-mem.v2.8.0. Refresh tool discovery after upgrading.
|
|
40
|
+
Retired proposal fields fail validation before provider work.
|
|
42
41
|
|
|
43
42
|
## Complete Remember request and safe replay
|
|
44
43
|
|
|
@@ -48,45 +47,31 @@ from densemem import RememberRequest
|
|
|
48
47
|
request: RememberRequest = {
|
|
49
48
|
"target_space": "team_shared",
|
|
50
49
|
"idempotency_key": retained_operation_key,
|
|
51
|
-
"evidence": [
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
],
|
|
59
|
-
"relationships": [
|
|
60
|
-
{
|
|
61
|
-
"ref": "durable-store",
|
|
62
|
-
"subject": {"name": "Dense-Mem", "entity_kind": "project"},
|
|
63
|
-
"predicate": {"proposed_key": "stores_durable_memory_in"},
|
|
64
|
-
"object": {"entity": {"name": "PostgreSQL", "entity_kind": "product"}},
|
|
65
|
-
"polarity": "+",
|
|
66
|
-
"evidence_indices": [0],
|
|
67
|
-
}
|
|
68
|
-
],
|
|
50
|
+
"evidence": [{
|
|
51
|
+
"content": "Dense-Mem stores durable memory in PostgreSQL.",
|
|
52
|
+
"source_type": "manual",
|
|
53
|
+
"source": "Python core example",
|
|
54
|
+
"metadata": {"origin": "python-example"},
|
|
55
|
+
}],
|
|
69
56
|
}
|
|
70
57
|
result = await client.remember(request)
|
|
71
58
|
~~~
|
|
72
59
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
one proposal citing every evidence item. The client does not infer Relationships.
|
|
60
|
+
Retain the complete request and key before sending. Replay the unchanged request
|
|
61
|
+
including target, metadata, and key to reconcile an unknown outcome. The client
|
|
62
|
+
never retries automatically or invents defaults. Legacy operation keys return
|
|
63
|
+
non-retryable retired_contract guidance without duplicating accepted Evidence.
|
|
78
64
|
|
|
79
|
-
Choose private for personal
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
65
|
+
Choose private for personal memory and team_shared for team knowledge.
|
|
66
|
+
Authentication selects the actual team, owner, and space. Every credential has
|
|
67
|
+
separate private memory. Team-read-only credentials may remember privately;
|
|
68
|
+
shared writes require current permission. The server never redirects a denied
|
|
69
|
+
shared write.
|
|
84
70
|
|
|
85
|
-
Inspect
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
provenance
|
|
89
|
-
establish new factual support. Trace requires a returned relationship_id.
|
|
71
|
+
Inspect Evidence outcomes, search states, and explicit warnings. Recall preserves
|
|
72
|
+
server order, equivalent IDs, Evidence conflicts, topic hints, and degradations.
|
|
73
|
+
Trace takes an evidence_id. Export accepts 1–500 Evidence IDs and returns a
|
|
74
|
+
versioned Evidence/provenance artifact with hashes and omission reporting.
|
|
90
75
|
|
|
91
76
|
## Exact private session ingestion
|
|
92
77
|
|
|
@@ -116,9 +101,10 @@ never authorization or private-space selection. The server must enable session
|
|
|
116
101
|
ingestion and the caller must have effective private write access. Every active
|
|
117
102
|
credential has its own private space, including team-read-only credentials;
|
|
118
103
|
SSO sessions use their user/team membership's separate private space. Discovery
|
|
119
|
-
hides unavailable tools. Remember requires its explicit target
|
|
120
|
-
Session
|
|
121
|
-
|
|
104
|
+
hides unavailable tools. Remember requires its explicit target.
|
|
105
|
+
Session acceptance is atomic across the request. The server selects exact
|
|
106
|
+
current-event excerpts with original offsets using session_evidence_excerpts_v2.
|
|
107
|
+
Empty extraction returns memory_outcome no_memory with search_state not_required.
|
|
122
108
|
|
|
123
109
|
## Failures and unknown outcomes
|
|
124
110
|
|
|
@@ -224,15 +210,16 @@ version matrix, and scripts/e2e.sh python_client for real server boundaries.
|
|
|
224
210
|
The example scripts require DENSEMEM_ENDPOINT, DENSEMEM_TOKEN and a retained
|
|
225
211
|
DENSEMEM_OPERATION_KEY. Keep the exact request unchanged when rerunning them.
|
|
226
212
|
|
|
227
|
-
The
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
213
|
+
The root [dependency exception policy](../../dependency-exceptions.json) is the
|
|
214
|
+
sole source of approved Go and Python vulnerability exceptions. It records exact
|
|
215
|
+
advisories, dependencies, versions, ADK prerequisites, reasons, approval links,
|
|
216
|
+
and exclusive UTC expiry dates. Both scanners resolve it from their repository
|
|
217
|
+
location, independently of the working directory; environment variables cannot
|
|
218
|
+
replace approvals. Missing or invalid policy fails closed. Empty sections grant
|
|
219
|
+
no exceptions. Scans still run and retain complete reports in output files or
|
|
220
|
+
console logs. Other findings, skipped dependencies and scanner failures remain
|
|
221
|
+
blocking, and enforcement resumes automatically at expiry. Remove an exception
|
|
222
|
+
sooner when compatible dependencies support its patched version.
|
|
236
223
|
|
|
237
224
|
The server prerelease and stable-release workflows call publish-python to build
|
|
238
225
|
and verify artifacts after their release gates pass, then upload through their own
|
|
@@ -293,9 +280,7 @@ or credential logging; caller-configured LangChain callbacks receive the normal
|
|
|
293
280
|
queries and returned Documents.
|
|
294
281
|
|
|
295
282
|
k defaults to 10 and accepts only integers from 1 through 50. recall_options
|
|
296
|
-
accepts
|
|
297
|
-
community_relationship_limit, valid_at, known_at, known_evidence_ids,
|
|
298
|
-
known_relationship_ids and expand_from_entity_ids. Values pass to the server
|
|
283
|
+
accepts topic_ids, topic_limit, valid_at, known_at, and known_evidence_ids. Values pass to the server
|
|
299
284
|
unchanged and are validated there. Query, limit, team and space overrides are
|
|
300
285
|
rejected; invocation-time recall overrides are unsupported. Options are copied
|
|
301
286
|
for each call. Server validation and authorization remain authoritative.
|
|
@@ -304,9 +289,9 @@ Each returned evidence item becomes one Document in server order. Document.id
|
|
|
304
289
|
is its evidence_id, and page_content is the exact server-bounded context without
|
|
305
290
|
additional truncation. Document.metadata["densemem"] contains evidence_id,
|
|
306
291
|
equivalent_evidence_ids, equivalents_truncated, space_kind, recall_id,
|
|
307
|
-
|
|
308
|
-
and suggested_actions.
|
|
309
|
-
|
|
292
|
+
related_topics, topics_truncated, conflicts, search_states, degradations
|
|
293
|
+
and suggested_actions. Topic hints describe the query and support bounded
|
|
294
|
+
follow-up through 1–10 unique topic_ids without descendant expansion. The current public
|
|
310
295
|
response exposes no document scores, ranks or source URLs, so none are invented.
|
|
311
296
|
Identical private and shared text remains separate Documents.
|
|
312
297
|
|
|
@@ -13,19 +13,10 @@ def make_request(key: str) -> RememberRequest:
|
|
|
13
13
|
"content": "Dense-Mem stores durable memory in PostgreSQL.",
|
|
14
14
|
"source_type": "manual",
|
|
15
15
|
"source": "Python core example",
|
|
16
|
-
"metadata": {"origin": "python-example"},
|
|
17
|
-
}
|
|
18
|
-
],
|
|
19
|
-
"relationships": [
|
|
20
|
-
{
|
|
21
|
-
"ref": "durable-store",
|
|
22
|
-
"subject": {"name": "Dense-Mem", "entity_kind": "project"},
|
|
23
|
-
"predicate": {"proposed_key": "stores_durable_memory_in"},
|
|
24
|
-
"object": {"entity": {"name": "PostgreSQL", "entity_kind": "product"}},
|
|
25
|
-
"polarity": "+",
|
|
26
|
-
"evidence_indices": [0],
|
|
16
|
+
"metadata": {"origin": "python-example", "revision": "1"},
|
|
27
17
|
}
|
|
28
18
|
],
|
|
19
|
+
|
|
29
20
|
}
|
|
30
21
|
|
|
31
22
|
|
|
@@ -35,9 +26,8 @@ async def run(client: DenseMemClient, key: str) -> RememberResult:
|
|
|
35
26
|
replay = await client.remember(request)
|
|
36
27
|
assert replay == result
|
|
37
28
|
await client.recall_memory({"query": "Dense-Mem durable memory PostgreSQL", "limit": 5})
|
|
38
|
-
for
|
|
39
|
-
|
|
40
|
-
await client.trace_memory({"relationship_id": split["relationship_id"]})
|
|
29
|
+
for item in result["evidence"]:
|
|
30
|
+
await client.trace_memory({"evidence_id": item["evidence_id"]})
|
|
41
31
|
return result
|
|
42
32
|
|
|
43
33
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "densemem"
|
|
7
|
-
version = "2.7.
|
|
7
|
+
version = "2.7.2rc4"
|
|
8
8
|
description = "Typed async and sync clients for Dense-Mem MCP"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11,<3.15"
|
|
@@ -21,7 +21,7 @@ classifiers = [
|
|
|
21
21
|
]
|
|
22
22
|
|
|
23
23
|
[project.urls]
|
|
24
|
-
Source = "https://github.com/markhuangai/dense-mem/tree/
|
|
24
|
+
Source = "https://github.com/markhuangai/dense-mem/tree/73080b340db41db481837f9ebd0ee6151321a96f"
|
|
25
25
|
Repository = "https://github.com/markhuangai/dense-mem"
|
|
26
26
|
Documentation = "https://github.com/markhuangai/dense-mem/wiki/Framework-Integrations"
|
|
27
27
|
Issues = "https://github.com/markhuangai/dense-mem/issues"
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
from datetime import UTC, date, datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load_policy(path: Path) -> list[dict]:
|
|
11
|
+
policy = json.loads(path.read_text())
|
|
12
|
+
if (
|
|
13
|
+
not isinstance(policy, dict)
|
|
14
|
+
or type(policy.get("schema_version")) is not int
|
|
15
|
+
or policy["schema_version"] != 1
|
|
16
|
+
or not isinstance(policy.get("go"), list)
|
|
17
|
+
or not isinstance(policy.get("python"), list)
|
|
18
|
+
):
|
|
19
|
+
raise ValueError("Expected schema_version 1 with go and python arrays")
|
|
20
|
+
for rule in policy["python"]:
|
|
21
|
+
if (
|
|
22
|
+
not isinstance(rule, dict)
|
|
23
|
+
or any(
|
|
24
|
+
not isinstance(rule.get(key), str) or not rule[key].strip()
|
|
25
|
+
for key in ("package", "version", "expires", "reason", "reference")
|
|
26
|
+
)
|
|
27
|
+
or not isinstance(rule.get("advisories"), list)
|
|
28
|
+
or not rule["advisories"]
|
|
29
|
+
or any(not isinstance(item, str) or not item.strip() for item in rule["advisories"])
|
|
30
|
+
or not isinstance(rule.get("requires"), dict)
|
|
31
|
+
or any(
|
|
32
|
+
not isinstance(rule["requires"].get(key), str) or not rule["requires"][key].strip()
|
|
33
|
+
for key in ("package", "version")
|
|
34
|
+
)
|
|
35
|
+
):
|
|
36
|
+
raise ValueError("Invalid Python dependency exception")
|
|
37
|
+
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", rule["expires"]):
|
|
38
|
+
raise ValueError("Exception expiry must be YYYY-MM-DD")
|
|
39
|
+
date.fromisoformat(rule["expires"])
|
|
40
|
+
return policy["python"]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def audit_status(
|
|
44
|
+
report: object, status: int, today: date, exceptions: list[dict]
|
|
45
|
+
) -> tuple[int, list[str]]:
|
|
46
|
+
if status not in (0, 1):
|
|
47
|
+
return status, [f"ERROR: reason=scanner_execution_failed scanner_status={status}"]
|
|
48
|
+
dependencies = report.get("dependencies") if isinstance(report, dict) else None
|
|
49
|
+
if not isinstance(dependencies, list) or not dependencies:
|
|
50
|
+
raise ValueError("Invalid pip-audit dependency report")
|
|
51
|
+
packages, findings = set(), []
|
|
52
|
+
for dependency in dependencies:
|
|
53
|
+
if (
|
|
54
|
+
not isinstance(dependency, dict)
|
|
55
|
+
or "skip_reason" in dependency
|
|
56
|
+
or not isinstance(dependency.get("name"), str)
|
|
57
|
+
or not dependency["name"]
|
|
58
|
+
or not isinstance(dependency.get("version"), str)
|
|
59
|
+
or not dependency["version"]
|
|
60
|
+
or not isinstance(dependency.get("vulns"), list)
|
|
61
|
+
):
|
|
62
|
+
raise ValueError("Invalid pip-audit dependency")
|
|
63
|
+
package = (dependency["name"], dependency["version"])
|
|
64
|
+
packages.add(package)
|
|
65
|
+
for vulnerability in dependency["vulns"]:
|
|
66
|
+
advisory = vulnerability.get("id") if isinstance(vulnerability, dict) else None
|
|
67
|
+
aliases = vulnerability.get("aliases", []) if isinstance(vulnerability, dict) else None
|
|
68
|
+
if (
|
|
69
|
+
not isinstance(advisory, str)
|
|
70
|
+
or not advisory
|
|
71
|
+
or not isinstance(aliases, list)
|
|
72
|
+
or any(not isinstance(alias, str) or not alias for alias in aliases)
|
|
73
|
+
):
|
|
74
|
+
raise ValueError("Invalid pip-audit vulnerability")
|
|
75
|
+
findings.append((package, advisory, {advisory, *aliases}))
|
|
76
|
+
if not findings:
|
|
77
|
+
return status, [f"reason=no_findings scanner_status={status}"]
|
|
78
|
+
approved, messages = True, []
|
|
79
|
+
for package, advisory, identities in findings:
|
|
80
|
+
candidates = [rule for rule in exceptions if identities.intersection(rule["advisories"])]
|
|
81
|
+
exact = [rule for rule in candidates if package == (rule["package"], rule["version"])]
|
|
82
|
+
environment = [
|
|
83
|
+
rule for rule in exact
|
|
84
|
+
if (rule["requires"]["package"], rule["requires"]["version"]) in packages
|
|
85
|
+
]
|
|
86
|
+
active = [rule for rule in environment if today < date.fromisoformat(rule["expires"])]
|
|
87
|
+
if status == 0:
|
|
88
|
+
reason = "scanner_status_inconsistent"
|
|
89
|
+
elif not candidates:
|
|
90
|
+
reason = "unapproved_advisory"
|
|
91
|
+
elif not exact:
|
|
92
|
+
reason = "dependency_mismatch"
|
|
93
|
+
elif not environment:
|
|
94
|
+
reason = "required_dependency_missing"
|
|
95
|
+
elif not active:
|
|
96
|
+
reason = "exception_expired"
|
|
97
|
+
else:
|
|
98
|
+
reason = "accepted_exception"
|
|
99
|
+
accepted = reason == "accepted_exception"
|
|
100
|
+
approved &= accepted
|
|
101
|
+
messages.append(
|
|
102
|
+
f"{'WARNING' if accepted else 'ERROR'}: advisory={advisory} "
|
|
103
|
+
f"dependency={package[0]}=={package[1]} reason={reason} scanner_status={status}"
|
|
104
|
+
)
|
|
105
|
+
return (0 if approved else 1), messages
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def main() -> int:
|
|
109
|
+
parser = argparse.ArgumentParser()
|
|
110
|
+
parser.add_argument("requirements", type=Path)
|
|
111
|
+
parser.add_argument("output", type=Path)
|
|
112
|
+
args = parser.parse_args()
|
|
113
|
+
policy_path = Path(__file__).resolve().parents[3] / "dependency-exceptions.json"
|
|
114
|
+
try:
|
|
115
|
+
exceptions = load_policy(policy_path)
|
|
116
|
+
except (OSError, ValueError) as error:
|
|
117
|
+
print(f"ERROR: policy={policy_path} reason=invalid_policy: {error}", file=sys.stderr)
|
|
118
|
+
return 1
|
|
119
|
+
print(f"policy={policy_path} scanner=pip-audit report={args.output}", file=sys.stderr)
|
|
120
|
+
for rule in exceptions:
|
|
121
|
+
print(
|
|
122
|
+
f"advisories={','.join(rule['advisories'])} "
|
|
123
|
+
f"dependency={rule['package']}=={rule['version']} expires={rule['expires']} "
|
|
124
|
+
f"requires={rule['requires']['package']}=={rule['requires']['version']} "
|
|
125
|
+
f"reason={rule['reason']} reference={rule['reference']}", file=sys.stderr,
|
|
126
|
+
)
|
|
127
|
+
result = subprocess.run(
|
|
128
|
+
[
|
|
129
|
+
sys.executable, "-m", "pip_audit", "-r", str(args.requirements),
|
|
130
|
+
"--no-deps", "--disable-pip", "--strict", "--format", "json",
|
|
131
|
+
],
|
|
132
|
+
stdout=subprocess.PIPE,
|
|
133
|
+
text=True,
|
|
134
|
+
check=False,
|
|
135
|
+
)
|
|
136
|
+
args.output.write_text(result.stdout)
|
|
137
|
+
sys.stdout.write(result.stdout)
|
|
138
|
+
if result.returncode not in (0, 1):
|
|
139
|
+
print(
|
|
140
|
+
f"ERROR: reason=scanner_execution_failed scanner_status={result.returncode} "
|
|
141
|
+
f"report={args.output}", file=sys.stderr,
|
|
142
|
+
)
|
|
143
|
+
return result.returncode
|
|
144
|
+
try:
|
|
145
|
+
status, messages = audit_status(
|
|
146
|
+
json.loads(result.stdout), result.returncode, datetime.now(UTC).date(), exceptions
|
|
147
|
+
)
|
|
148
|
+
except ValueError as error:
|
|
149
|
+
print(
|
|
150
|
+
f"ERROR: pip-audit report rejected: {error}; scanner_status={result.returncode} "
|
|
151
|
+
f"report={args.output}", file=sys.stderr,
|
|
152
|
+
)
|
|
153
|
+
return 1
|
|
154
|
+
for message in messages:
|
|
155
|
+
print(f"{message} report={args.output}", file=sys.stderr)
|
|
156
|
+
return status
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
if __name__ == "__main__":
|
|
160
|
+
raise SystemExit(main())
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
set -euo pipefail
|
|
3
3
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
|
4
4
|
OUTPUT="${1:?Provide a new output directory}"
|
|
5
|
+
if [[ $# != 1 && $# != 3 ]]; then
|
|
6
|
+
printf 'Usage: matrix.sh OUTPUT [PYTHON SDK]\n' >&2
|
|
7
|
+
exit 2
|
|
8
|
+
fi
|
|
5
9
|
[[ ! -e "$OUTPUT" ]] || { printf 'Output already exists\n' >&2; exit 1; }
|
|
6
10
|
mkdir -p "$OUTPUT"
|
|
7
11
|
CONTAINER=""
|
|
@@ -15,15 +19,17 @@ EPOCH="$(git -C "$ROOT" show -s --format=%ct HEAD)"
|
|
|
15
19
|
|
|
16
20
|
run_case() {
|
|
17
21
|
local version="$1" sdk="$2" destination="$3" status
|
|
22
|
+
printf 'Checking Python %s (%s SDK)\n' "$version" "$sdk"
|
|
18
23
|
mkdir -p "$destination"
|
|
19
|
-
CONTAINER="$(docker create --workdir /
|
|
24
|
+
CONTAINER="$(docker create --workdir /workspace/packages/python \
|
|
20
25
|
-e "SOURCE_DATE_EPOCH=$EPOCH" "python:$version-slim" \
|
|
21
26
|
bash -c 'set -euo pipefail
|
|
22
27
|
apt-get update -qq
|
|
23
28
|
apt-get install -y --no-install-recommends git
|
|
24
|
-
exec bash /
|
|
29
|
+
exec bash /workspace/packages/python/scripts/check.sh --output /results --sdk "$1"
|
|
25
30
|
' -- "$sdk")"
|
|
26
|
-
docker cp "$ROOT/packages/python/." "$CONTAINER:/
|
|
31
|
+
docker cp "$ROOT/packages/python/." "$CONTAINER:/workspace/packages/python"
|
|
32
|
+
docker cp "$ROOT/dependency-exceptions.json" "$CONTAINER:/workspace/dependency-exceptions.json"
|
|
27
33
|
if docker start --attach "$CONTAINER"; then
|
|
28
34
|
status="$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")"
|
|
29
35
|
else
|
|
@@ -33,8 +39,13 @@ run_case() {
|
|
|
33
39
|
docker inspect --format '{{json .Image}}' "$CONTAINER" > "$destination/image.json"
|
|
34
40
|
docker rm "$CONTAINER" >/dev/null
|
|
35
41
|
CONTAINER=""
|
|
42
|
+
printf 'Python %s (%s SDK) completed with status %s\n' "$version" "$sdk" "$status"
|
|
36
43
|
return "$status"
|
|
37
44
|
}
|
|
45
|
+
if [[ $# == 3 ]]; then
|
|
46
|
+
run_case "$2" "$3" "$OUTPUT/$2-$3"
|
|
47
|
+
exit
|
|
48
|
+
fi
|
|
38
49
|
for version in 3.11 3.12 3.13 3.14; do
|
|
39
50
|
run_case "$version" latest "$OUTPUT/$version"
|
|
40
51
|
done
|
|
@@ -10,14 +10,14 @@ from .errors import (
|
|
|
10
10
|
)
|
|
11
11
|
from .requests import (
|
|
12
12
|
Evidence,
|
|
13
|
+
ExportRequest,
|
|
13
14
|
RecallRequest,
|
|
14
|
-
RelationshipProposal,
|
|
15
15
|
RememberRequest,
|
|
16
16
|
SessionEvent,
|
|
17
17
|
SessionRequest,
|
|
18
18
|
TraceRequest,
|
|
19
19
|
)
|
|
20
|
-
from .results import RecallResult, RememberResult, SessionResult, TraceResult
|
|
20
|
+
from .results import ExportResult, RecallResult, RememberResult, SessionResult, TraceResult
|
|
21
21
|
|
|
22
22
|
__all__ = [
|
|
23
23
|
"DenseMemAuthenticationError",
|
|
@@ -31,7 +31,8 @@ __all__ = [
|
|
|
31
31
|
"Evidence",
|
|
32
32
|
"RecallRequest",
|
|
33
33
|
"RecallResult",
|
|
34
|
-
"
|
|
34
|
+
"ExportRequest",
|
|
35
|
+
"ExportResult",
|
|
35
36
|
"RememberRequest",
|
|
36
37
|
"RememberResult",
|
|
37
38
|
"SessionEvent",
|
|
@@ -25,8 +25,8 @@ from .errors import (
|
|
|
25
25
|
DenseMemTimeoutError,
|
|
26
26
|
DenseMemTransportError,
|
|
27
27
|
)
|
|
28
|
-
from .requests import RecallRequest, RememberRequest, SessionRequest, TraceRequest
|
|
29
|
-
from .results import RecallResult, RememberResult, SessionResult, TraceResult
|
|
28
|
+
from .requests import ExportRequest, RecallRequest, RememberRequest, SessionRequest, TraceRequest
|
|
29
|
+
from .results import ExportResult, RecallResult, RememberResult, SessionResult, TraceResult
|
|
30
30
|
|
|
31
31
|
_operation_token: ContextVar[str] = ContextVar("densemem_operation_token", default="")
|
|
32
32
|
|
|
@@ -108,6 +108,11 @@ class DenseMemClient:
|
|
|
108
108
|
async def trace_memory(self, request: TraceRequest) -> TraceResult:
|
|
109
109
|
return cast(TraceResult, await self._execute("trace_memory", cast(dict[str, Any], request)))
|
|
110
110
|
|
|
111
|
+
async def export_memory_pack(self, request: ExportRequest) -> ExportResult:
|
|
112
|
+
return cast(
|
|
113
|
+
ExportResult, await self._execute("export_memory_pack", cast(dict[str, Any], request))
|
|
114
|
+
)
|
|
115
|
+
|
|
111
116
|
async def _execute(self, method: str, request: dict[str, Any] | None = None) -> Any:
|
|
112
117
|
payload = copy.deepcopy(request)
|
|
113
118
|
token = ""
|
|
@@ -308,3 +313,6 @@ class SyncDenseMemClient:
|
|
|
308
313
|
|
|
309
314
|
def trace_memory(self, request: TraceRequest) -> TraceResult:
|
|
310
315
|
return self._run(lambda: self._async.trace_memory(request))
|
|
316
|
+
|
|
317
|
+
def export_memory_pack(self, request: ExportRequest) -> ExportResult:
|
|
318
|
+
return self._run(lambda: self._async.export_memory_pack(request))
|
|
@@ -107,8 +107,7 @@ def _documents(result: RecallResult, k: int) -> list[Document]:
|
|
|
107
107
|
raise ValueError
|
|
108
108
|
context = {key: public[key] for key in RecallResult.__annotations__ if key != "results"}
|
|
109
109
|
for key in (
|
|
110
|
-
"
|
|
111
|
-
"related_communities",
|
|
110
|
+
"related_topics",
|
|
112
111
|
"conflicts",
|
|
113
112
|
"degradations",
|
|
114
113
|
"suggested_actions",
|
|
@@ -117,10 +116,12 @@ def _documents(result: RecallResult, k: int) -> list[Document]:
|
|
|
117
116
|
not isinstance(item, dict) for item in context[key]
|
|
118
117
|
):
|
|
119
118
|
raise ValueError
|
|
119
|
+
if type(result["topics_truncated"]) is not bool:
|
|
120
|
+
raise ValueError
|
|
120
121
|
states = result["search_states"]
|
|
121
122
|
if not isinstance(states, dict) or any(
|
|
122
123
|
states[key] not in ("current", "not_required", "pending", "failed")
|
|
123
|
-
for key in ("evidence",
|
|
124
|
+
for key in ("evidence",)
|
|
124
125
|
):
|
|
125
126
|
raise ValueError
|
|
126
127
|
for degradation in result["degradations"]:
|